Merge pull request #348 from code-dot-org/molly/exit-on-memory-outage

Force system exit on out of memory error

Author: molly-moen

org-code-javabuilder/lib/src/main/java/org/code/javabuilder/ExceptionHandler.java

@@ -32,6 +32,18 @@ public void handle(Throwable e) {
       return;
     }
 
+    // We should also exit on an OutOfMemoryError.
+    // This means something is wrong with our memory usage
+    // and is not likely to be recoverable.
+    if (e instanceof OutOfMemoryError) {
+      final InternalServerException error =
+          new InternalServerException(InternalExceptionKey.UNKNOWN_ERROR, e);
+      LoggerUtils.logSevereError(error);
+      LambdaUtils.safelySendMessage(this.outputAdapter, error.getExceptionMessage(), true);
+      this.systemExitHelper.exit(LambdaErrorCodes.OUT_OF_MEMORY_ERROR_CODE);
+      return;
+    }
+
     // Internal server exceptions are caused by us (essentially an HTTP 5xx error). Log and notify
     // the user.
     if (e instanceof InternalServerException || e instanceof InternalServerRuntimeException) {

org-code-javabuilder/lib/src/main/java/org/code/javabuilder/LambdaErrorCodes.java

@@ -7,4 +7,5 @@ private LambdaErrorCodes() {
 
   public static final int TEMP_DIRECTORY_CLEANUP_ERROR_CODE = 10;
   public static final int LOW_DISK_SPACE_ERROR_CODE = 50;
+  public static final int OUT_OF_MEMORY_ERROR_CODE = 60;
 }

org-code-javabuilder/lib/src/main/java/org/code/javabuilder/LambdaRequestHandler.java

@@ -43,9 +43,6 @@ public class LambdaRequestHandler implements RequestHandler<Map<String, String>,
   private static final int TIMEOUT_WARNING_MS = 20000;
   private static final int TIMEOUT_CLEANUP_BUFFER_MS = 5000;
   private static final String LAMBDA_ID = UUID.randomUUID().toString();
-  // This is an error code we made up to signal that available disk space is less than 50%.
-  // It may be used in tracking on Cloud Watch.
-  private static final int LOW_DISK_SPACE_ERROR_CODE = 50;
   private static final String CONTENT_BUCKET_NAME = System.getenv("CONTENT_BUCKET_NAME");
   private static final String CONTENT_BUCKET_URL = System.getenv("CONTENT_BUCKET_URL");
   private static final String API_ENDPOINT = System.getenv("API_ENDPOINT");
@@ -314,7 +311,7 @@ private void shutDown(
     if ((double) f.getUsableSpace() / f.getTotalSpace() < 0.5) {
       // The current project holds a lock on too many resources. Force the JVM to quit in
       // order to release the resources for the next use of the container.
-      System.exit(LOW_DISK_SPACE_ERROR_CODE);
+      System.exit(LambdaErrorCodes.LOW_DISK_SPACE_ERROR_CODE);
     }
 
     this.isSessionInitialized = false;

org-code-javabuilder/protocol/src/main/java/org/code/protocol/LoggerUtils.java

@@ -108,6 +108,10 @@ public static void logTrackingException(Throwable e) {
     Logger.getLogger(MAIN_LOGGER).warning(eventData.toString());
   }
 
+  public static void logInfo(String info) {
+    Logger.getLogger(MAIN_LOGGER).info(info);
+  }
+
   private static void sendDiskSpaceLogs(String type) {
     File f = Paths.get(System.getProperty("java.io.tmpdir")).toFile();
     JSONObject eventData = new JSONObject();