Re: Optimize type streaming

[Jan Hubicka <hubicka at ucw dot cz>]

> 
> We have reverted the patch for now but I note that at least the piece
> below is a step backward from doing the compare in the on-disk
> format.

Why it is step backward from compare in the on-dist format? All the information
is still streamed, just not duplicated.  Since we explicitly stream main variant
pointer, we will have all the comparsion done in on-disk format if we was able
to do it.
> 
> So I'd rather push back the whole change a bit until I find the time
> to explore that (as it complicates the code quite a bit).

It indeed adds some of complexity and tree streaming is your domain ;)
I will push out the sanity checkng part (that is actually bit more important for me
than the actual reduction in streaming) to a type verifier. There are more invariants
to look into and more bugs to fix.

I did some more experiments in this direction (not streaming data that is
redundant).  In similar way I removed streaming sizes/modes for decls (since
they are 99% copied from trees) and streaming the items that are usually shared
with main variant, but not always (fields).  With copying done here in streamer
it works quite nicely and do save space in the stream/time needed to read back. 

Honza
> 
> Richard.
> 
> > Honza
> > 
> > Index: lto-streamer-in.c
> > ===================================================================
> > --- lto-streamer-in.c	(revision 212098)
> > +++ lto-streamer-in.c	(working copy)
> > @@ -1182,6 +1182,58 @@ lto_read_tree (struct lto_input_block *i
> >  }
> >  
> >  
> > +/* Copy fields that are not streamed but copied from other nodes.  */
> > +static void
> > +lto_copy_fields_not_streamed (tree t)
> > +{
> > +  if (TYPE_P (t) && TYPE_MAIN_VARIANT (t) != t)
> > +    {
> > +      tree mv = TYPE_MAIN_VARIANT (t);
> > +
> > +      if (COMPLETE_TYPE_P (t))
> > +	{
> > +	  TYPE_SIZE (t) = TYPE_SIZE (mv);
> > +	  TYPE_SIZE_UNIT (t) = TYPE_SIZE_UNIT (mv);
> > +	}
> > +      TYPE_ATTRIBUTES (t) = TYPE_ATTRIBUTES (mv);
> > +
> > +      if (CODE_CONTAINS_STRUCT (TREE_CODE (t), TS_TYPE_NON_COMMON))
> > +	{
> > +	  if (TREE_CODE (t) == ENUMERAL_TYPE && COMPLETE_TYPE_P (t))
> > +	    TYPE_VALUES (t) = TYPE_VALUES (mv);
> > +	  else if (TREE_CODE (t) == ARRAY_TYPE)
> > +	    TYPE_DOMAIN (t) = TYPE_DOMAIN (mv);
> > +
> > +          if (RECORD_OR_UNION_TYPE_P (t) && COMPLETE_TYPE_P (t))
> > +	    TYPE_VFIELD (t) = TYPE_VFIELD (mv);
> > +	  else if ((TREE_CODE (t) == ENUMERAL_TYPE && COMPLETE_TYPE_P (t))
> > +		   || TREE_CODE (t) == INTEGER_TYPE
> > +		   || TREE_CODE (t) == BOOLEAN_TYPE
> > +		   || TREE_CODE (t) == REAL_TYPE
> > +		   || TREE_CODE (t) == FIXED_POINT_TYPE)
> > +	    TYPE_MIN_VALUE (t) = TYPE_MIN_VALUE (mv);
> > +
> > +	  if (TREE_CODE (t) == METHOD_TYPE)
> > +	    TYPE_METHOD_BASETYPE (t) = TYPE_METHOD_BASETYPE (mv);
> > +	  else if (RECORD_OR_UNION_TYPE_P (t) && COMPLETE_TYPE_P (t))
> > +	    TYPE_METHODS (t) = TYPE_METHODS (mv);
> > +	  else if (TREE_CODE (t) == OFFSET_TYPE)
> > +	    TYPE_OFFSET_BASETYPE (t) = TYPE_OFFSET_BASETYPE (mv);
> > +	  else if (TREE_CODE (t) == ARRAY_TYPE)
> > +	    TYPE_ARRAY_MAX_SIZE (t) = TYPE_ARRAY_MAX_SIZE (mv);
> > +	  else if ((TREE_CODE (t) == ENUMERAL_TYPE && COMPLETE_TYPE_P (t))
> > +		   || TREE_CODE (t) == INTEGER_TYPE
> > +		   || TREE_CODE (t) == BOOLEAN_TYPE
> > +		   || TREE_CODE (t) == REAL_TYPE
> > +		   || TREE_CODE (t) == FIXED_POINT_TYPE)
> > +	    TYPE_MAX_VALUE (t) = TYPE_MAX_VALUE (mv);
> > +
> > +	  if (RECORD_OR_UNION_TYPE_P (t) && COMPLETE_TYPE_P (t))
> > +	    TYPE_BINFO (t) = TYPE_BINFO (mv);
> > +	}
> > +    }
> > +}
> > +
> >  /* Populate the reader cache with trees materialized from the SCC
> >     following in the IB, DATA_IN stream.  */
> >  
> > @@ -1194,6 +1246,7 @@ lto_input_scc (struct lto_input_block *i
> >    unsigned size = streamer_read_uhwi (ib);
> >    hashval_t scc_hash = streamer_read_uhwi (ib);
> >    unsigned scc_entry_len = 1;
> > +  unsigned from = data_in->reader_cache->nodes.length ();
> >  
> >    if (size == 1)
> >      {
> > @@ -1233,6 +1286,12 @@ lto_input_scc (struct lto_input_block *i
> >  	}
> >      }
> >  
> > +  /* Copy fileds we do not stream before unification so we can compare them
> > +     without being worried if they are already initialized.  */
> > +  for (unsigned i = 0; i < size; ++i)
> > +    lto_copy_fields_not_streamed
> > +       (streamer_tree_cache_get_tree (data_in->reader_cache, from + i));
> > +
> >    *len = size;
> >    *entry_len = scc_entry_len;
> >    return scc_hash;
> > Index: lto/lto.c
> > ===================================================================
> > --- lto/lto.c	(revision 212114)
> > +++ lto/lto.c	(working copy)
> > @@ -1050,58 +1050,6 @@ lto_register_function_decl_in_symtab (st
> >  			 decl, get_resolution (data_in, ix));
> >  }
> >  
> > -/* Copy fields that are not streamed but copied from other nodes.  */
> > -static void
> > -lto_copy_fields_not_streamed (tree t)
> > -{
> > -  if (TYPE_P (t) && TYPE_MAIN_VARIANT (t) != t)
> > -    {
> > -      tree mv = TYPE_MAIN_VARIANT (t);
> > -
> > -      if (COMPLETE_TYPE_P (t))
> > -	{
> > -	  TYPE_SIZE (t) = TYPE_SIZE (mv);
> > -	  TYPE_SIZE_UNIT (t) = TYPE_SIZE_UNIT (mv);
> > -	}
> > -      TYPE_ATTRIBUTES (t) = TYPE_ATTRIBUTES (mv);
> > -
> > -      if (CODE_CONTAINS_STRUCT (TREE_CODE (t), TS_TYPE_NON_COMMON))
> > -	{
> > -	  if (TREE_CODE (t) == ENUMERAL_TYPE && COMPLETE_TYPE_P (t))
> > -	    TYPE_VALUES (t) = TYPE_VALUES (mv);
> > -	  else if (TREE_CODE (t) == ARRAY_TYPE)
> > -	    TYPE_DOMAIN (t) = TYPE_DOMAIN (mv);
> > -
> > -          if (RECORD_OR_UNION_TYPE_P (t) && COMPLETE_TYPE_P (t))
> > -	    TYPE_VFIELD (t) = TYPE_VFIELD (mv);
> > -	  else if ((TREE_CODE (t) == ENUMERAL_TYPE && COMPLETE_TYPE_P (t))
> > -		   || TREE_CODE (t) == INTEGER_TYPE
> > -		   || TREE_CODE (t) == BOOLEAN_TYPE
> > -		   || TREE_CODE (t) == REAL_TYPE
> > -		   || TREE_CODE (t) == FIXED_POINT_TYPE)
> > -	    TYPE_MIN_VALUE (t) = TYPE_MIN_VALUE (mv);
> > -
> > -	  if (TREE_CODE (t) == METHOD_TYPE)
> > -	    TYPE_METHOD_BASETYPE (t) = TYPE_METHOD_BASETYPE (mv);
> > -	  else if (RECORD_OR_UNION_TYPE_P (t) && COMPLETE_TYPE_P (t))
> > -	    TYPE_METHODS (t) = TYPE_METHODS (mv);
> > -	  else if (TREE_CODE (t) == OFFSET_TYPE)
> > -	    TYPE_OFFSET_BASETYPE (t) = TYPE_OFFSET_BASETYPE (mv);
> > -	  else if (TREE_CODE (t) == ARRAY_TYPE)
> > -	    TYPE_ARRAY_MAX_SIZE (t) = TYPE_ARRAY_MAX_SIZE (mv);
> > -	  else if ((TREE_CODE (t) == ENUMERAL_TYPE && COMPLETE_TYPE_P (t))
> > -		   || TREE_CODE (t) == INTEGER_TYPE
> > -		   || TREE_CODE (t) == BOOLEAN_TYPE
> > -		   || TREE_CODE (t) == REAL_TYPE
> > -		   || TREE_CODE (t) == FIXED_POINT_TYPE)
> > -	    TYPE_MAX_VALUE (t) = TYPE_MAX_VALUE (mv);
> > -
> > -	  if (RECORD_OR_UNION_TYPE_P (t) && COMPLETE_TYPE_P (t))
> > -	    TYPE_BINFO (t) = TYPE_BINFO (mv);
> > -	}
> > -    }
> > -}
> > -
> >  /* For the type T re-materialize it in the type variant list and
> >     the pointer/reference-to chains.  */
> >  
> > @@ -1958,19 +1906,13 @@ lto_read_decls (struct lto_file_decl_dat
> >  		  || streamer_handle_as_builtin_p (first)))
> >  	    continue;
> >  
> > -	  /* Copy fileds we do not stream before unification so we can compare them
> > -	     without being worried if they are already initialized.  */
> > -	  for (unsigned i = 0; i < len; ++i)
> > -	    lto_copy_fields_not_streamed
> > -	       (streamer_tree_cache_get_tree (data_in->reader_cache, from + i));
> > -
> >  	  /* Try to unify the SCC with already existing ones.  */
> >  	  if (!flag_ltrans
> >  	      && unify_scc (data_in->reader_cache, from,
> >  			    len, scc_entry_len, scc_hash))
> >  	    continue;
> >  
> >  	  /* Do remaining fixup tasks for prevailing nodes.  */
> >  	  bool seen_type = false;
> >  	  for (unsigned i = 0; i < len; ++i)
> >  	    {
> > 
> > 
> 
> -- 
> Richard Biener <rguenther@suse.de>
> SUSE / SUSE Labs
> SUSE LINUX Products GmbH - Nuernberg - AG Nuernberg - HRB 16746
> GF: Jeff Hawn, Jennifer Guild, Felix Imend"orffer