[Patch] Overflows in array allocation.

[Ralph Loader <suckfish at ihug dot co dot nz>]

Hi,

The two array allocation functions in prims.cc have overflow problems.

_Jv_NewObjectArray wasn't checking for overflow when calculating the
number of bytes for an array.

_Jv_NewPrimArray was correctly checking for overflowing a size_t, but
then truncated to jint - which is a problem on machines with 64 bit
size_t.

The following patch fixes both to ensure that the number of bytes does
not overflow a jint, i.e., ((1<<31)-1), since the lower level allocation
functions take a jint (or a jsize, which is the same thing).

A better fix might be change the allocation functions to take size_t
rather than jint / jsize.  That would change quite a few places, and I'm
not sure if / what requirements there are not to change the ABI of the
functions & objects involved.  I can do this if it's wanted.

Ralph.

2003-09-24  Ralph Loader  <suckfish@ihug.co.nz>

	* prims.cc (_Jv_NewObjectArray): Make sure byte size doesn't
	overflow a jint.
	(_Jv_NewPrimArray): Check for overflowing a jint, replacing a
	check for overflowing size_t, since the lower level functions
	take a jint.
	* testsuite/libjava.lang/newarray_overflow.java:  New file.
	* testsuite/libjava.lang/newarray_overflow.out:  New file.

Index: libjava/prims.cc
===================================================================
RCS file: /cvsroot/gcc/gcc/libjava/prims.cc,v
retrieving revision 1.82
diff -u -u -r1.82 prims.cc
--- libjava/prims.cc	20 Aug 2003 14:32:15 -0000	1.82
+++ libjava/prims.cc	24 Sep 2003 04:27:05 -0000
@@ -70,8 +70,10 @@
 // around for use if we run out of memory.
 static java::lang::OutOfMemoryError *no_memory;
 
-// Largest representable size_t.
-#define SIZE_T_MAX ((size_t) (~ (size_t) 0))
+// Number of bytes in largest array object we create.  This could be
+// increased to the largest size_t value, so long as the appropriate
+// functions are changed to take a size_t argument instead of jint.
+#define MAX_OBJECT_SIZE ((1<<31) - 1)
 
 static const char *no_properties[] = { NULL };
 
@@ -481,6 +483,11 @@
   // Ensure that elements pointer is properly aligned.
   jobjectArray obj = NULL;
   size_t size = (size_t) elements (obj);
+  // Check for overflow.
+  if (__builtin_expect ((size_t) count > 
+			(MAX_OBJECT_SIZE - 1 - size) / sizeof (jobject), false))
+    throw no_memory;
+
   size += count * sizeof (jobject);
 
   jclass klass = _Jv_GetArrayClass (elementClass,
@@ -516,7 +523,7 @@
 
   // Check for overflow.
   if (__builtin_expect ((size_t) count > 
-			(SIZE_T_MAX - size) / elsize, false))
+			(MAX_OBJECT_SIZE - size) / elsize, false))
     throw no_memory;
 
   jclass klass = _Jv_GetArrayClass (eltype, 0);
--- /dev/null	2003-09-16 01:40:47.000000000 +1200
+++ libjava/testsuite/libjava.lang/newarray_overflow.java	2003-09-24 16:12:32.000000000 +1200
@@ -0,0 +1,88 @@
+/* This test checks for two slightly different overflow scenarios in
+ * array allocation.
+ *
+ * The first is that the number of bytes needed for an array size
+ * overflows on a 32 bit machine.
+ *
+ * The second is that on a 64 machine, the number of bytes silently
+ * gets truncated, resulting in too small an object being
+ * allocated.  */
+
+class newarray_overflow
+{
+  static boolean failed = false;
+
+  static void int_check()
+  {
+    int[] x;
+    try
+      {
+	x = new int [1 << 30];
+      }
+    catch (OutOfMemoryError e)
+      {
+	return;
+      }
+    /* If we really get away with it (64 bit machine), that's cool.  */
+    if (x == null) {
+      System.err.println ("int check: new returned null.");
+      failed = true;
+      return;
+    }
+    try
+      {
+	// Only check a few places so we don't thrash too badly.
+	for (int i = 0; i < x.length; i += (1 << 24))
+	  if (x[i] != 0)
+	    failed = true;
+      }
+    catch (Throwable e)
+      {
+	System.err.print ("int check: ");
+	System.err.println (e);
+	failed = true;
+      }
+  }
+
+  static void object_check()
+  {
+    Object[] x;
+    try
+      {
+	x = new Object [1 << 30];
+	System.err.println ("Alloc succeeded.");
+	System.err.println (x);
+      }
+    catch (OutOfMemoryError e)
+      {
+	return;
+      }
+    /* If we really get away with it (64 bit machine), that's cool.  */
+    if (x == null) {
+      System.err.println ("Object check: new returned null.");
+      failed = true;
+      return;
+    }
+    try
+      {
+	for (int i = 0; i < x.length; i += (1 << 24))
+	  if (x[i] != null)
+	    failed = true;
+      }
+    catch (Throwable e)
+      {
+	System.err.print ("Object check: ");
+	System.err.println (e);
+	failed = true;
+      }
+  }
+
+  public static void main (String[] ignore)
+  {
+    int_check();
+    object_check();
+
+    if (!failed)
+      System.out.println ("ok");
+  }
+}
--- /dev/null	2003-09-16 01:40:47.000000000 +1200
+++ libjava/testsuite/libjava.lang/newarray_overflow.out	2003-09-24 16:04:31.000000000 +1200
@@ -0,0 +1 @@
+ok