[Patch] PR 31228, Fix close-on-exec race.

[David Daney <ddaney at avtrex dot com>]

This is an ugly problem:

If Runtime.exec() forks between when a file descriptor is created and 
when we can set FD_CLOEXEC on it, the descriptor will leak to the 
subprocess.

The fix is to close all possible file descriptors before doing the exec 
system call.

I don't really like the fix, as it typically adds over 1000 system calls 
for each execed process.  The only bright side is that it is no longer 
necessary to call _Jv_platform_close_on_exec, so I removed it.  I am a 
little uneasy about that also, but thought what the heck...

Tested on x86_64-pc-linux (FC6) with no regressions in make -k check in 
libjava.

OK to commit?

I hope someone can think of a better way to fix the race.  If only there 
were some way to set the default value of the FD_CLOEXEC flag...

2007-03-19  David Daney  <ddaney@avtrex.com)

   PR libgcj/31228
   * configure.ac: Add checks for getrlimit and sys/resource.h.
   * include/posix.h (_Jv_platform_close_on_exec): Remove.
   * include/config.h.in: Regenerate.
   * configure: Regenerate.
   * gnu/java/nio/channels/natFileChannelPosix.cc (open): Remove call to
   _Jv_platform_close_on_exec;
   * gnu/java/net/natPlainSocketImplPosix.cc (create): Likewise.
   (accept): Likewise.
   * gnu/java/net/natPlainDatagramSocketImplPosix.cc (create):Likewise.
   * java/lang/natPosixProcess.cc: Include sys/resource.h.
   (nativeSpawn): Close all file descriptors.  Don't set FD_CLOEXEC on
   pipes.

Index: configure.ac
===================================================================
--- configure.ac	(revision 123033)
+++ configure.ac	(working copy)
@@ -1006,10 +1006,10 @@ else
 		   access stat lstat mkdir rename rmdir unlink utime chmod readlink \
 		   nl_langinfo setlocale \
 		   inet_pton uname inet_ntoa \
-		   fork execvp pipe sigaction ftruncate mmap \
+		   fork execvp getrlimit pipe sigaction ftruncate mmap \
 		   getifaddrs])
    AC_CHECK_FUNCS(inet_aton inet_addr, break)
-   AC_CHECK_HEADERS(execinfo.h unistd.h dlfcn.h)
+   AC_CHECK_HEADERS(execinfo.h unistd.h dlfcn.h sys/resource.h)
    # Do an additional check on dld, HP-UX for example has dladdr in libdld.sl
    AC_CHECK_LIB(dl, dladdr, [
        AC_DEFINE(HAVE_DLADDR, 1, [Define if you have dladdr()])], [
Index: include/posix.h
===================================================================
--- include/posix.h	(revision 123033)
+++ include/posix.h	(working copy)
@@ -98,15 +98,6 @@ extern jlong _Jv_platform_nanotime ();
 extern void _Jv_platform_initialize (void);
 extern void _Jv_platform_initProperties (java::util::Properties*);
 
-inline void
-_Jv_platform_close_on_exec (jint fd)
-{
-  // Ignore errors.
-  ::fcntl (fd, F_SETFD, FD_CLOEXEC);
-}
-
-#undef fcntl
-
 #ifdef JV_HASH_SYNCHRONIZATION
 #ifndef HAVE_USLEEP_DECL
 extern "C" int usleep (useconds_t useconds);
Index: include/config.h.in
===================================================================
--- include/config.h.in	(revision 123033)
+++ include/config.h.in	(working copy)
@@ -127,6 +127,9 @@
 /* Define to 1 if you have the `getpwuid_r' function. */
 #undef HAVE_GETPWUID_R
 
+/* Define to 1 if you have the `getrlimit' function. */
+#undef HAVE_GETRLIMIT
+
 /* Define to 1 if you have the `gettimeofday' function. */
 #undef HAVE_GETTIMEOFDAY
 
@@ -316,6 +319,9 @@
 /* Define to 1 if you have the <sys/ioctl.h> header file. */
 #undef HAVE_SYS_IOCTL_H
 
+/* Define to 1 if you have the <sys/resource.h> header file. */
+#undef HAVE_SYS_RESOURCE_H
+
 /* Define to 1 if you have the <sys/rw_lock.h> header file. */
 #undef HAVE_SYS_RW_LOCK_H
 
Index: configure
===================================================================
--- configure	(revision 123033)
+++ configure	(working copy)
@@ -9602,12 +9602,13 @@ else
 
 
 
+
 for ac_func in strerror ioctl select fstat open fsync sleep opendir \
                    gmtime_r localtime_r readdir_r getpwuid_r getcwd \
 		   access stat lstat mkdir rename rmdir unlink utime chmod readlink \
 		   nl_langinfo setlocale \
 		   inet_pton uname inet_ntoa \
-		   fork execvp pipe sigaction ftruncate mmap \
+		   fork execvp getrlimit pipe sigaction ftruncate mmap \
 		   getifaddrs
 do
 as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
@@ -10063,7 +10064,8 @@ done
 
 
 
-for ac_header in execinfo.h unistd.h dlfcn.h
+
+for ac_header in execinfo.h unistd.h dlfcn.h sys/resource.h
 do
 as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh`
 if eval "test \"\${$as_ac_Header+set}\" = set"; then
Index: gnu/java/nio/channels/natFileChannelPosix.cc
===================================================================
--- gnu/java/nio/channels/natFileChannelPosix.cc	(revision 123033)
+++ gnu/java/nio/channels/natFileChannelPosix.cc	(working copy)
@@ -178,8 +178,6 @@ FileChannelImpl::open (jstring path, jin
       throw new ::java::io::FileNotFoundException (msg->toString ());
     }
 
-  _Jv_platform_close_on_exec (fd);
-
   return fd;
 }
 
Index: gnu/java/net/natPlainSocketImplPosix.cc
===================================================================
--- gnu/java/net/natPlainSocketImplPosix.cc	(revision 123033)
+++ gnu/java/net/natPlainSocketImplPosix.cc	(working copy)
@@ -72,8 +72,6 @@ gnu::java::net::PlainSocketImpl::create 
       throw new ::java::io::IOException (JvNewStringUTF (strerr));
     }
 
-  _Jv_platform_close_on_exec (sock);
-
   // We use native_fd in place of fd here.  From leaving fd null we avoid
   // the double close problem in FileDescriptor.finalize.
   native_fd = sock;
@@ -285,8 +283,6 @@ gnu::java::net::PlainSocketImpl::accept 
   if (new_socket < 0)
     goto error;
 
-  _Jv_platform_close_on_exec (new_socket);
-
   jbyteArray raddr;
   jint rport;
   if (u.address.sin_family == AF_INET)
Index: gnu/java/net/natPlainDatagramSocketImplPosix.cc
===================================================================
--- gnu/java/net/natPlainDatagramSocketImplPosix.cc	(revision 123033)
+++ gnu/java/net/natPlainDatagramSocketImplPosix.cc	(working copy)
@@ -83,8 +83,6 @@ gnu::java::net::PlainDatagramSocketImpl:
       throw new ::java::net::SocketException (JvNewStringUTF (strerr));
     }
 
-  _Jv_platform_close_on_exec (sock);
-
   // We use native_fd in place of fd here.  From leaving fd null we avoid
   // the double close problem in FileDescriptor.finalize.
   native_fd = sock;
Index: java/lang/natPosixProcess.cc
===================================================================
--- java/lang/natPosixProcess.cc	(revision 123033)
+++ java/lang/natPosixProcess.cc	(working copy)
@@ -17,6 +17,9 @@ details.  */
 #include <fcntl.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>
+#endif
 #include <signal.h>
 #include <string.h>
 #include <stdlib.h>
@@ -352,7 +355,31 @@ java::lang::PosixProcess::nativeSpawn ()
 		  _exit (127);
 		}
 	    }
-
+          // Make sure all file descriptors are closed.  In
+          // multi-threaded programs, there is a race between when a
+          // descriptor is obtained, when we can set FD_CLOEXEC, and
+          // fork().  If the fork occurs before FD_CLOEXEC is set, the
+          // descriptor would leak to the execed process if we did not
+          // manually close it.  So that is what we do.  Since we
+          // close all the descriptors, it is redundant to set
+          // FD_CLOEXEC on them elsewhere.
+          int max_fd;
+#ifdef HAVE_GETRLIMIT
+          rlimit rl;
+          int rv = getrlimit(RLIMIT_NOFILE, &rl);
+          if (rv == 0)
+            max_fd = rl.rlim_cur - 1;
+          else
+            max_fd = 1024 - 1;
+#else
+          max_fd = 1024 - 1;
+#endif
+          while(max_fd > 2)
+            {
+              if (max_fd != msgp[1])
+                close (max_fd);
+              max_fd--;
+            }
 	  // Make sure that SIGCHLD is unblocked for the new process.
 	  sigset_t mask;
 	  sigemptyset (&mask);
@@ -438,12 +465,4 @@ java::lang::PosixProcess::nativeSpawn ()
 
   myclose (msgp[0]);
   cleanup (args, env, path);
-
-  if (exception == NULL)
-    {
-      fcntl (outp[1], F_SETFD, FD_CLOEXEC);
-      fcntl (inp[0], F_SETFD, FD_CLOEXEC);
-      if (! redirect)
-	fcntl (errp[0], F_SETFD, FD_CLOEXEC);
-    }
 }