Fix for 'Too many open files' errors

Donn Terry donnte@microsoft.com
Thu Jun 29 09:33:00 GMT 2000


For the record: My system exhibits the EMFILE variant
on the problem, and the proposed patch fixes the problem
just fine.  Thanks.

(I literally was about to write you about it when I saw
this mail, so I just tried it instead.)

Donn

P.S.  (Please apply :-) if you haven't by the time this
reaches you.)

> -----Original Message-----
> From: Zack Weinberg [ mailto:zack@wolery.cumb.org ]
> Sent: Wednesday, June 28, 2000 12:15 AM
> To: RDBrown@mira.net; lauras@softhome.net
> Cc: gcc-patches@gcc.gnu.org
> Subject: Fix for 'Too many open files' errors
> 
> 
> This patch makes cpp detect an EMFILE or ENFILE error return from
> open() and close all its cached file descriptors.  This should make it
> possible to bootstrap again on systems with a small limit on open
> files per process.  (We can look into cleverer cache-replacement
> policies later; the important bit right now is to get things working.)
> 
> The patch also adds the ability to report on headers that could use
> multiple include guards.  Right now this is triggered by -H; I'm open
> to user-interface suggestions.  At the end of preprocessing you get a
> list of all the headers that were read only once but that don't have
> multiple include guards.  You can't blindly add guards to all files so
> listed, but it gives you a place to start.
> 
> Bootstrapped and tested on i386-linux.  I'll hold off committing it
> until I hear a success report for one of the affected systems.
> 
> [For those unclear on the difference: EMFILE is for the per-process
> limit on open files, ENFILE is the system-wide limit.]
> 
> zw
> 
> 	* cppfiles.c (open_include_file): If open(2) returns EMFILE or 
> 	ENFILE,	close all cached file descriptors and try again.
> 	(_cpp_execute_include): Keep a count of the number of times
> 	each header is included.
> 	(close_cached_fd): New function.
> 	* cpphash.h (struct include_file): Rename before to
> 	include_count; all users updated.  Make include_count and sysp
> 	unsigned short.
> 	* cppinit.c (cpp_finish): If -H, report headers that could use 
> 	reinclude guards.
> 	(report_missing_guard): New function.
> 
> ===================================================================
> Index: cppfiles.c
> --- cppfiles.c	2000/06/21 23:08:17	1.67
> +++ cppfiles.c	2000/06/28 07:08:41
> @@ -60,6 +60,7 @@ static ssize_t read_with_read	PARAMS ((c
>  static ssize_t read_file	PARAMS ((cpp_buffer *, int, ssize_t));
>  
>  static void destroy_include_file_node	PARAMS 
> ((splay_tree_value));
> +static int close_cached_fd	PARAMS ((splay_tree_node, void *));
>  
>  #if 0
>  static void hack_vms_include_specification PARAMS ((char *));
> @@ -87,6 +88,20 @@ destroy_include_file_node (v)
>      }
>  }
>  
> +static int
> +close_cached_fd (n, dummy)
> +     splay_tree_node n;
> +     void *dummy ATTRIBUTE_UNUSED;
> +{
> +  struct include_file *f = (struct include_file *)n->value;
> +  if (f && f->fd != -1)
> +    {
> +      close (f->fd);
> +      f->fd = -1;
> +    }
> +  return 0;
> +}
> +
>  void
>  _cpp_init_include_table (pfile)
>       cpp_reader *pfile;
> @@ -153,6 +168,8 @@ open_include_file (pfile, filename)
>       ourselves.
>  
>       Special case: the empty string is translated to stdin.  */
> + retry:
> +
>    if (filename[0] == '\0')
>      fd = 0;
>    else
> @@ -167,6 +184,21 @@ open_include_file (pfile, filename)
>  		     filename);
>  	}
>  #endif
> +      if (0
> +#ifdef EMFILE
> +	  || errno == EMFILE
> +#endif
> +#ifdef ENFILE
> +	  || errno == ENFILE
> +#endif
> +	  )
> +	{
> +	  /* Too many files open.  Close all cached file descriptors and
> +	     try again.  */
> +	  splay_tree_foreach (pfile->all_include_files, 
> close_cached_fd, 0);
> +	  goto retry;
> +	}
> +
>        /* Nonexistent or inaccessible file.  Create a 
> negative node for it.  */
>        if (nd)
>  	{
> @@ -185,7 +217,7 @@ open_include_file (pfile, filename)
>      {
>        file = xnew (struct include_file);
>        file->cmacro = 0;
> -      file->before = 0;
> +      file->include_count = 0;
>        file->sysp = 0;
>        file->foundhere = 0;
>        file->name = xstrdup (filename);
> @@ -367,9 +399,9 @@ _cpp_execute_include (pfile, f, len, no_
>  	return;
>  
>        /* For -M, add the file to the dependencies on its 
> first inclusion. */
> -      if (!inc->before && PRINT_THIS_DEP (pfile, angle_brackets))
> +      if (!inc->include_count && PRINT_THIS_DEP (pfile, 
> angle_brackets))
>  	deps_add_dep (pfile->deps, inc->name);
> -      inc->before = 1;
> +      inc->include_count++;
>  
>        /* Handle -H option.  */
>        if (CPP_OPTION (pfile, print_include_names))
> ===================================================================
> Index: cpphash.h
> --- cpphash.h	2000/06/21 18:33:51	1.54
> +++ cpphash.h	2000/06/28 07:08:41
> @@ -62,8 +62,8 @@ struct include_file
>  				/* location in search path 
> where file was
>  				   found, for #include_next */
>    int fd;			/* file descriptor possibly 
> open on file */
> -  unsigned char before;		/* file has been 
> included before */
> -  unsigned char sysp;		/* file is a system header */
> +  unsigned short include_count;	/* number of times file 
> has been read */
> +  unsigned short sysp;		/* file is a system header */
>  };
>  
>  /* The cmacro works like this: If it's NULL, the file is to be
> ===================================================================
> Index: cppinit.c
> --- cppinit.c	2000/06/21 18:33:51	1.83
> +++ cppinit.c	2000/06/28 07:08:42
> @@ -227,6 +227,7 @@ static int opt_comp			
> PARAMS ((const voi
>  #endif
>  static int parse_option			PARAMS ((const char *));
>  static int handle_option		PARAMS ((cpp_reader *, 
> int, char **));
> +static int report_missing_guard		PARAMS 
> ((splay_tree_node, void *));
>  
>  /* Fourth argument to append_include_chain: chain to use */
>  enum { QUOTE = 0, BRACKET, SYSTEM, AFTER };
> @@ -1003,6 +1004,27 @@ cpp_start_read (pfile, print, fname)
>    return 1;
>  }
>  
> +static int
> +report_missing_guard (n, b)
> +     splay_tree_node n;
> +     void *b;
> +{
> +  struct include_file *f = (struct include_file *) n->value;
> +  int *bannerp = (int *)b;
> +
> +  if (f && f->cmacro == 0 && f->include_count == 1)
> +    {
> +      if (*bannerp == 0)
> +	{
> +	  fputs (_("Multiple include guards may be useful 
> for:\n"), stderr);
> +	  *bannerp = 1;
> +	}
> +      fputs (f->name, stderr);
> +      putc ('\n', stderr);
> +    }
> +  return 0;
> +}
> +
>  /* This is called at the end of preprocessing.  It pops the
>     last buffer and writes dependency output.  It should also
>     clear macro definitions, such that you could call cpp_start_read
> @@ -1054,6 +1076,14 @@ cpp_finish (pfile, print)
>        cpp_output_tokens (pfile, print);
>        if (ferror (print->outf) || fclose (print->outf))
>  	cpp_notice_from_errno (pfile, CPP_OPTION (pfile, out_fname));
> +    }
> +
> +  /* Report on headers that could use multiple include guards.  */
> +  if (CPP_OPTION (pfile, print_include_names))
> +    {
> +      int banner = 0;
> +      splay_tree_foreach (pfile->all_include_files, 
> report_missing_guard,
> +			  (void *) &banner);
>      }
>  }
>  
> 


More information about the Gcc-patches mailing list