Heap dump should not be generated on handled exceptions

Patch by Paulo Motta; Reviewed by Isaac Reath, Stefan Miklosovic for CASSANDRA-20974

Author: pauloricardomg

CHANGES.txt

@@ -1,4 +1,5 @@
 5.0.6
+ * Heap dump should not be generated on handled exceptions (CASSANDRA-20974)
  * Throw RTE instead of FSError when RTE is thrown from FileUtis.write in TOCComponent (CASSANDRA-20917)
  * Upgrade jackson-dataformat-yaml to 2.19.2 and snakeyaml to 2.1 (CASSANDRA-18875)
  * Represent complex settings as JSON on system_views.settings table (CASSANDRA-20827)

src/java/org/apache/cassandra/utils/HeapUtils.java

@@ -128,10 +128,6 @@ public static String maybeCreateHeapDump()
 
                     return fullPath;
                 }
-                else
-                {
-                    logger.debug("Heap dump creation on uncaught exceptions is disabled.");
-                }
             }
             catch (Throwable e)
             {

src/java/org/apache/cassandra/utils/JVMStabilityInspector.java

@@ -75,7 +75,7 @@ public static void uncaughtException(Thread thread, Throwable t)
             if (t2 != t && (t2 instanceof FSError || t2 instanceof CorruptSSTableException))
                 logger.error("Exception in thread {}", thread, t2);
         }
-        JVMStabilityInspector.inspectThrowable(t);
+        inspectThrowable(t, JVMStabilityInspector::inspectDiskError, true);
     }
 
     /**
@@ -86,12 +86,12 @@ public static void uncaughtException(Thread thread, Throwable t)
      */
     public static void inspectThrowable(Throwable t) throws OutOfMemoryError
     {
-        inspectThrowable(t, JVMStabilityInspector::inspectDiskError);
+        inspectThrowable(t, JVMStabilityInspector::inspectDiskError, false);
     }
 
     public static void inspectCommitLogThrowable(Throwable t)
     {
-        inspectThrowable(t, JVMStabilityInspector::inspectCommitLogError);
+        inspectThrowable(t, JVMStabilityInspector::inspectCommitLogError, false);
     }
 
     private static void inspectDiskError(Throwable t)
@@ -102,7 +102,7 @@ else if (t instanceof FSError)
             FileUtils.handleFSError((FSError) t);
     }
 
-    public static void inspectThrowable(Throwable t, Consumer<Throwable> fn) throws OutOfMemoryError
+    public static void inspectThrowable(Throwable t, Consumer<Throwable> fn, boolean isUncaughtException) throws OutOfMemoryError
     {
         boolean isUnstable = false;
         if (t instanceof OutOfMemoryError)
@@ -136,7 +136,22 @@ else if (t instanceof UnrecoverableIllegalStateException)
         }
 
         // Anything other than an OOM, we should try and heap dump to capture what's going on if configured to do so
-        HeapUtils.maybeCreateHeapDump();
+        if (isUncaughtException)
+        {
+            try
+            {
+                // Avoid entering maybeCreateHeapDump unless the setting is enabled
+                // since that will attempt to grab an expensive lock
+                if (DatabaseDescriptor.getDumpHeapOnUncaughtException())
+                {
+                    HeapUtils.maybeCreateHeapDump();
+                }
+            }
+            catch (Throwable sub)
+            {
+                t.addSuppressed(sub);
+            }
+        }
 
         if (t instanceof InterruptedException)
             throw new UncheckedInterruptedException((InterruptedException) t);
@@ -167,7 +182,7 @@ else if (t instanceof UnrecoverableIllegalStateException)
         }
 
         if (t.getCause() != null)
-            inspectThrowable(t.getCause(), fn);
+            inspectThrowable(t.getCause(), fn, isUncaughtException);
     }
 
     private static final Set<String> FORCE_HEAP_OOM_IGNORE_SET = ImmutableSet.of("Java heap space", "GC Overhead limit exceeded");