[PATCH 0/6, OpenACC, libgomp] Async re-work

[Thomas Schwinge]

Hi Chung-Lin!

On Fri, 14 Dec 2018 22:28:58 +0800, Chung-Lin Tang <chunglin_tang@mentor.com> wrote:
> On 2018/12/13 11:51 PM, Thomas Schwinge wrote:
> > On Thu, 13 Dec 2018 23:28:49 +0800, Chung-Lin Tang<chunglin_tang@mentor.com>  wrote:
> >> On 2018/12/7 6:26 AM, Julian Brown wrote:
> >>> On Thu, 6 Dec 2018 22:22:46 +0000
> >>> Julian Brown<julian@codesourcery.com>  wrote:
> >>>
> >>>> On Thu, 6 Dec 2018 21:42:14 +0100
> >>>> Thomas Schwinge<thomas@codesourcery.com>  wrote:
> >>>>
> >>>>> [...]
> >>>>> ..., where the "Invalid read of size 8" happens, and which
> >>>>> eventually would try to "free (tgt)" again, via
> >>>>> libgomp/target.c:gomp_unmap_tgt:
> >>>>>
> >>>>>       attribute_hidden void
> >>>>>       gomp_unmap_tgt (struct target_mem_desc *tgt)
> >>>>>       {
> >>>>>         /* Deallocate on target the tgt->tgt_start .. tgt->tgt_end
> >>>>> region.  */ if (tgt->tgt_end)
> >>>>>           gomp_free_device_memory (tgt->device_descr, tgt->to_free);
> >>>>>       
> >>>>>         free (tgt->array);
> >>>>>         free (tgt);
> >>>>>       }
> >>>>>
> >>>>> Is the "free (tgt)" in libgomp/target.c:gomp_unmap_vars_async wrong,
> >>>>> or something else?
> 
> I think I understand the problem now. In gomp_unmap_vars_async(), in the case of
> tgt->list_count == 0 (i.e. no map arguments at all) the code should simply free the tgt
> and return, while the code in goacc_async_copyout_unmap_vars() didn't handle this case
> and always scheduled an asynchronous free of the tgt later, causing that valgrind error
> you see.
> 
> I am still testing the attached patch, but I think it is the right fix: I reviewed what I
> wrote and it seemed the way I organized things into a goacc_async_copyout_unmap_vars() routine,
> including the hackish refcount++, etc. is simply unneeded. I have deleted those stuff
> and consolidated things back into gomp_unmap_vars_async().
> 
> I'll update the whole patches later after complete testing, the attached is the patch atop
> of the prior async patches. (the small program you gave above does pass valgrind now)

Thanks, confirmed.


Grüße
 Thomas


> diff -ru trunk-orig/libgomp/oacc-async.c trunk-work/libgomp/oacc-async.c
> --- trunk-orig/libgomp/oacc-async.c	2018-12-14 21:06:06.649794724 +0800
> +++ trunk-work/libgomp/oacc-async.c	2018-12-14 22:11:29.252251925 +0800
> @@ -238,31 +238,6 @@
>    thr->default_async = async;
>  }
>  
> -static void
> -goacc_async_unmap_tgt (void *ptr)
> -{
> -  struct target_mem_desc *tgt = (struct target_mem_desc *) ptr;
> -
> -  if (tgt->refcount > 1)
> -    tgt->refcount--;
> -  else
> -    gomp_unmap_tgt (tgt);
> -}
> -
> -attribute_hidden void
> -goacc_async_copyout_unmap_vars (struct target_mem_desc *tgt,
> -				struct goacc_asyncqueue *aq)
> -{
> -  struct gomp_device_descr *devicep = tgt->device_descr;
> -
> -  /* Increment reference to delay freeing of device memory until callback
> -     has triggered.  */
> -  tgt->refcount++;
> -  gomp_unmap_vars_async (tgt, true, aq);
> -  devicep->openacc.async.queue_callback_func (aq, goacc_async_unmap_tgt,
> -					      (void *) tgt);
> -}
> -
>  attribute_hidden void
>  goacc_async_free (struct gomp_device_descr *devicep,
>  		  struct goacc_asyncqueue *aq, void *ptr)
> diff -ru trunk-orig/libgomp/oacc-int.h trunk-work/libgomp/oacc-int.h
> --- trunk-orig/libgomp/oacc-int.h	2018-12-14 21:06:06.649794724 +0800
> +++ trunk-work/libgomp/oacc-int.h	2018-12-14 22:11:43.379947915 +0800
> @@ -104,8 +104,6 @@
>  
>  void goacc_init_asyncqueues (struct gomp_device_descr *);
>  bool goacc_fini_asyncqueues (struct gomp_device_descr *);
> -void goacc_async_copyout_unmap_vars (struct target_mem_desc *,
> -				     struct goacc_asyncqueue *);
>  void goacc_async_free (struct gomp_device_descr *, struct goacc_asyncqueue *,
>  		       void *);
>  struct goacc_asyncqueue *get_goacc_asyncqueue (int);
> diff -ru trunk-orig/libgomp/oacc-mem.c trunk-work/libgomp/oacc-mem.c
> --- trunk-orig/libgomp/oacc-mem.c	2018-12-14 21:06:06.649794724 +0800
> +++ trunk-work/libgomp/oacc-mem.c	2018-12-14 22:10:08.325998369 +0800
> @@ -911,7 +911,7 @@
>        else
>  	{
>  	  goacc_aq aq = get_goacc_asyncqueue (async);
> -	  goacc_async_copyout_unmap_vars (t, aq);
> +	  gomp_unmap_vars_async (t, true, aq);
>  	}
>      }
>  
> diff -ru trunk-orig/libgomp/oacc-parallel.c trunk-work/libgomp/oacc-parallel.c
> --- trunk-orig/libgomp/oacc-parallel.c	2018-12-14 21:06:06.649794724 +0800
> +++ trunk-work/libgomp/oacc-parallel.c	2018-12-14 22:09:51.918353575 +0800
> @@ -245,7 +245,7 @@
>      {
>        acc_dev->openacc.async.exec_func (tgt_fn, mapnum, hostaddrs, devaddrs,
>  					dims, tgt, aq);
> -      goacc_async_copyout_unmap_vars (tgt, aq);
> +      gomp_unmap_vars_async (tgt, true, aq);
>      }
>  }
>  
> diff -ru trunk-orig/libgomp/target.c trunk-work/libgomp/target.c
> --- trunk-orig/libgomp/target.c	2018-12-14 21:06:06.653794622 +0800
> +++ trunk-work/libgomp/target.c	2018-12-14 20:42:03.629154346 +0800
> @@ -1072,6 +1072,17 @@
>    return is_tgt_unmapped;
>  }
>  
> +static void
> +gomp_unref_tgt (void *ptr)
> +{
> +  struct target_mem_desc *tgt = (struct target_mem_desc *) ptr;
> +
> +  if (tgt->refcount > 1)
> +    tgt->refcount--;
> +  else
> +    gomp_unmap_tgt (tgt);
> +}
> +
>  /* Unmap variables described by TGT.  If DO_COPYFROM is true, copy relevant
>     variables back from device to host: if it is false, it is assumed that this
>     has been done already.  */
> @@ -1130,10 +1141,11 @@
>  	gomp_remove_var (devicep, k);
>      }
>  
> -  if (tgt->refcount > 1)
> -    tgt->refcount--;
> +  if (aq)
> +    devicep->openacc.async.queue_callback_func (aq, gomp_unref_tgt,
> +						(void *) tgt);
>    else
> -    gomp_unmap_tgt (tgt);
> +    gomp_unref_tgt ((void *) tgt);
>  
>    gomp_mutex_unlock (&devicep->lock);
>  }