libgomp/oacc-mem.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2015 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "openacc.h"
  30 #include "config.h"
  31 #include "libgomp.h"
  32 #include "gomp-constants.h"
  33 #include "oacc-int.h"
  34 #include <stdint.h>
  35 #include <assert.h>
  36
  37 /* Return block containing [H->S), or NULL if not contained.  The device lock
  38    for DEV must be locked on entry, and remains locked on exit.  */
  39
  40 static splay_tree_key
  41 lookup_host (struct gomp_device_descr *dev, void *h, size_t s)
  42 {
  43   struct splay_tree_key_s node;
  44   splay_tree_key key;
  45
  46   node.host_start = (uintptr_t) h;
  47   node.host_end = (uintptr_t) h + s;
  48
  49   key = splay_tree_lookup (&dev->mem_map, &node);
  50
  51   return key;
  52 }
  53
  54 /* Return block containing [D->S), or NULL if not contained.
  55    The list isn't ordered by device address, so we have to iterate
  56    over the whole array.  This is not expected to be a common
  57    operation.  The device lock associated with TGT must be locked on entry, and
  58    remains locked on exit.  */
  59
  60 static splay_tree_key
  61 lookup_dev (struct target_mem_desc *tgt, void *d, size_t s)
  62 {
  63   int i;
  64   struct target_mem_desc *t;
  65
  66   if (!tgt)
  67     return NULL;
  68
  69   for (t = tgt; t != NULL; t = t->prev)
  70     {
  71       if (t->tgt_start <= (uintptr_t) d && t->tgt_end >= (uintptr_t) d + s)
  72         break;
  73     }
  74
  75   if (!t)
  76     return NULL;
  77
  78   for (i = 0; i < t->list_count; i++)
  79     {
  80       void * offset;
  81
  82       splay_tree_key k = &t->array[i].key;
  83       offset = d - t->tgt_start + k->tgt_offset;
  84
  85       if (k->host_start + offset <= (void *) k->host_end)
  86         return k;
  87     }
  88
  89   return NULL;
  90 }
  91
  92 /* OpenACC is silent on how memory exhaustion is indicated.  We return
  93    NULL.  */
  94
  95 void *
  96 acc_malloc (size_t s)
  97 {
  98   if (!s)
  99     return NULL;
 100
 101   goacc_lazy_initialize ();
 102
 103   struct goacc_thread *thr = goacc_thread ();
 104
 105   assert (thr->dev);
 106
 107   return thr->dev->alloc_func (thr->dev->target_id, s);
 108 }
 109
 110 /* OpenACC 2.0a (3.2.16) doesn't specify what to do in the event
 111    the device address is mapped. We choose to check if it mapped,
 112    and if it is, to unmap it. */
 113 void
 114 acc_free (void *d)
 115 {
 116   splay_tree_key k;
 117
 118   if (!d)
 119     return;
 120
 121   struct goacc_thread *thr = goacc_thread ();
 122
 123   assert (thr && thr->dev);
 124
 125   struct gomp_device_descr *acc_dev = thr->dev;
 126
 127   gomp_mutex_lock (&acc_dev->lock);
 128
 129   /* We don't have to call lazy open here, as the ptr value must have
 130      been returned by acc_malloc.  It's not permitted to pass NULL in
 131      (unless you got that null from acc_malloc).  */
 132   if ((k = lookup_dev (acc_dev->openacc.data_environ, d, 1)))
 133     {
 134       void *offset;
 135
 136       offset = d - k->tgt->tgt_start + k->tgt_offset;
 137
 138       gomp_mutex_unlock (&acc_dev->lock);
 139
 140       acc_unmap_data ((void *)(k->host_start + offset));
 141     }
 142   else
 143     gomp_mutex_unlock (&acc_dev->lock);
 144
 145   acc_dev->free_func (acc_dev->target_id, d);
 146 }
 147
 148 void
 149 acc_memcpy_to_device (void *d, void *h, size_t s)
 150 {
 151   /* No need to call lazy open here, as the device pointer must have
 152      been obtained from a routine that did that.  */
 153   struct goacc_thread *thr = goacc_thread ();
 154
 155   assert (thr && thr->dev);
 156
 157   thr->dev->host2dev_func (thr->dev->target_id, d, h, s);
 158 }
 159
 160 void
 161 acc_memcpy_from_device (void *h, void *d, size_t s)
 162 {
 163   /* No need to call lazy open here, as the device pointer must have
 164      been obtained from a routine that did that.  */
 165   struct goacc_thread *thr = goacc_thread ();
 166
 167   assert (thr && thr->dev);
 168
 169   thr->dev->dev2host_func (thr->dev->target_id, h, d, s);
 170 }
 171
 172 /* Return the device pointer that corresponds to host data H.  Or NULL
 173    if no mapping.  */
 174
 175 void *
 176 acc_deviceptr (void *h)
 177 {
 178   splay_tree_key n;
 179   void *d;
 180   void *offset;
 181
 182   goacc_lazy_initialize ();
 183
 184   struct goacc_thread *thr = goacc_thread ();
 185   struct gomp_device_descr *dev = thr->dev;
 186
 187   gomp_mutex_lock (&dev->lock);
 188
 189   n = lookup_host (dev, h, 1);
 190
 191   if (!n)
 192     {
 193       gomp_mutex_unlock (&dev->lock);
 194       return NULL;
 195     }
 196
 197   offset = h - n->host_start;
 198
 199   d = n->tgt->tgt_start + n->tgt_offset + offset;
 200
 201   gomp_mutex_unlock (&dev->lock);
 202
 203   return d;
 204 }
 205
 206 /* Return the host pointer that corresponds to device data D.  Or NULL
 207    if no mapping.  */
 208
 209 void *
 210 acc_hostptr (void *d)
 211 {
 212   splay_tree_key n;
 213   void *h;
 214   void *offset;
 215
 216   goacc_lazy_initialize ();
 217
 218   struct goacc_thread *thr = goacc_thread ();
 219   struct gomp_device_descr *acc_dev = thr->dev;
 220
 221   gomp_mutex_lock (&acc_dev->lock);
 222
 223   n = lookup_dev (acc_dev->openacc.data_environ, d, 1);
 224
 225   if (!n)
 226     {
 227       gomp_mutex_unlock (&acc_dev->lock);
 228       return NULL;
 229     }
 230
 231   offset = d - n->tgt->tgt_start + n->tgt_offset;
 232
 233   h = n->host_start + offset;
 234
 235   gomp_mutex_unlock (&acc_dev->lock);
 236
 237   return h;
 238 }
 239
 240 /* Return 1 if host data [H,+S] is present on the device.  */
 241
 242 int
 243 acc_is_present (void *h, size_t s)
 244 {
 245   splay_tree_key n;
 246
 247   if (!s || !h)
 248     return 0;
 249
 250   goacc_lazy_initialize ();
 251
 252   struct goacc_thread *thr = goacc_thread ();
 253   struct gomp_device_descr *acc_dev = thr->dev;
 254
 255   gomp_mutex_lock (&acc_dev->lock);
 256
 257   n = lookup_host (acc_dev, h, s);
 258
 259   if (n && ((uintptr_t)h < n->host_start
 260             || (uintptr_t)h + s > n->host_end
 261             || s > n->host_end - n->host_start))
 262     n = NULL;
 263
 264   gomp_mutex_unlock (&acc_dev->lock);
 265
 266   return n != NULL;
 267 }
 268
 269 /* Create a mapping for host [H,+S] -> device [D,+S] */
 270
 271 void
 272 acc_map_data (void *h, void *d, size_t s)
 273 {
 274   struct target_mem_desc *tgt;
 275   size_t mapnum = 1;
 276   void *hostaddrs = h;
 277   void *devaddrs = d;
 278   size_t sizes = s;
 279   unsigned short kinds = GOMP_MAP_ALLOC;
 280
 281   goacc_lazy_initialize ();
 282
 283   struct goacc_thread *thr = goacc_thread ();
 284   struct gomp_device_descr *acc_dev = thr->dev;
 285
 286   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 287     {
 288       if (d != h)
 289         gomp_fatal ("cannot map data on shared-memory system");
 290
 291       tgt = gomp_map_vars (NULL, 0, NULL, NULL, NULL, NULL, true,
 292                            GOMP_MAP_VARS_OPENACC);
 293     }
 294   else
 295     {
 296       struct goacc_thread *thr = goacc_thread ();
 297
 298       if (!d || !h || !s)
 299         gomp_fatal ("[%p,+%d]->[%p,+%d] is a bad map",
 300                     (void *)h, (int)s, (void *)d, (int)s);
 301
 302       gomp_mutex_lock (&acc_dev->lock);
 303
 304       if (lookup_host (acc_dev, h, s))
 305         {
 306           gomp_mutex_unlock (&acc_dev->lock);
 307           gomp_fatal ("host address [%p, +%d] is already mapped", (void *)h,
 308                       (int)s);
 309         }
 310
 311       if (lookup_dev (thr->dev->openacc.data_environ, d, s))
 312         {
 313           gomp_mutex_unlock (&acc_dev->lock);
 314           gomp_fatal ("device address [%p, +%d] is already mapped", (void *)d,
 315                       (int)s);
 316         }
 317
 318       gomp_mutex_unlock (&acc_dev->lock);
 319
 320       tgt = gomp_map_vars (acc_dev, mapnum, &hostaddrs, &devaddrs, &sizes,
 321                            &kinds, true, GOMP_MAP_VARS_OPENACC);
 322     }
 323
 324   gomp_mutex_lock (&acc_dev->lock);
 325   tgt->prev = acc_dev->openacc.data_environ;
 326   acc_dev->openacc.data_environ = tgt;
 327   gomp_mutex_unlock (&acc_dev->lock);
 328 }
 329
 330 void
 331 acc_unmap_data (void *h)
 332 {
 333   struct goacc_thread *thr = goacc_thread ();
 334   struct gomp_device_descr *acc_dev = thr->dev;
 335
 336   /* No need to call lazy open, as the address must have been mapped.  */
 337
 338   size_t host_size;
 339
 340   gomp_mutex_lock (&acc_dev->lock);
 341
 342   splay_tree_key n = lookup_host (acc_dev, h, 1);
 343   struct target_mem_desc *t;
 344
 345   if (!n)
 346     {
 347       gomp_mutex_unlock (&acc_dev->lock);
 348       gomp_fatal ("%p is not a mapped block", (void *)h);
 349     }
 350
 351   host_size = n->host_end - n->host_start;
 352
 353   if (n->host_start != (uintptr_t) h)
 354     {
 355       gomp_mutex_unlock (&acc_dev->lock);
 356       gomp_fatal ("[%p,%d] surrounds %p",
 357                   (void *) n->host_start, (int) host_size, (void *) h);
 358     }
 359
 360   t = n->tgt;
 361
 362   if (t->refcount == 2)
 363     {
 364       struct target_mem_desc *tp;
 365
 366       /* This is the last reference, so pull the descriptor off the
 367          chain. This avoids gomp_unmap_vars via gomp_unmap_tgt from
 368          freeing the device memory. */
 369       t->tgt_end = 0;
 370       t->to_free = 0;
 371
 372       for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 373            tp = t, t = t->prev)
 374         if (n->tgt == t)
 375           {
 376             if (tp)
 377               tp->prev = t->prev;
 378             else
 379               acc_dev->openacc.data_environ = t->prev;
 380
 381             break;
 382           }
 383     }
 384
 385   gomp_mutex_unlock (&acc_dev->lock);
 386
 387   gomp_unmap_vars (t, true);
 388 }
 389
 390 #define FLAG_PRESENT (1 << 0)
 391 #define FLAG_CREATE (1 << 1)
 392 #define FLAG_COPY (1 << 2)
 393
 394 static void *
 395 present_create_copy (unsigned f, void *h, size_t s)
 396 {
 397   void *d;
 398   splay_tree_key n;
 399
 400   if (!h || !s)
 401     gomp_fatal ("[%p,+%d] is a bad range", (void *)h, (int)s);
 402
 403   goacc_lazy_initialize ();
 404
 405   struct goacc_thread *thr = goacc_thread ();
 406   struct gomp_device_descr *acc_dev = thr->dev;
 407
 408   gomp_mutex_lock (&acc_dev->lock);
 409
 410   n = lookup_host (acc_dev, h, s);
 411   if (n)
 412     {
 413       /* Present. */
 414       d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 415
 416       if (!(f & FLAG_PRESENT))
 417         {
 418           gomp_mutex_unlock (&acc_dev->lock);
 419           gomp_fatal ("[%p,+%d] already mapped to [%p,+%d]",
 420                       (void *)h, (int)s, (void *)d, (int)s);
 421         }
 422       if ((h + s) > (void *)n->host_end)
 423         {
 424           gomp_mutex_unlock (&acc_dev->lock);
 425           gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 426         }
 427
 428       gomp_mutex_unlock (&acc_dev->lock);
 429     }
 430   else if (!(f & FLAG_CREATE))
 431     {
 432       gomp_mutex_unlock (&acc_dev->lock);
 433       gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 434     }
 435   else
 436     {
 437       struct target_mem_desc *tgt;
 438       size_t mapnum = 1;
 439       unsigned short kinds;
 440       void *hostaddrs = h;
 441
 442       if (f & FLAG_COPY)
 443         kinds = GOMP_MAP_TO;
 444       else
 445         kinds = GOMP_MAP_ALLOC;
 446
 447       gomp_mutex_unlock (&acc_dev->lock);
 448
 449       tgt = gomp_map_vars (acc_dev, mapnum, &hostaddrs, NULL, &s, &kinds, true,
 450                            GOMP_MAP_VARS_OPENACC);
 451
 452       gomp_mutex_lock (&acc_dev->lock);
 453
 454       d = tgt->to_free;
 455       tgt->prev = acc_dev->openacc.data_environ;
 456       acc_dev->openacc.data_environ = tgt;
 457
 458       gomp_mutex_unlock (&acc_dev->lock);
 459     }
 460
 461   return d;
 462 }
 463
 464 void *
 465 acc_create (void *h, size_t s)
 466 {
 467   return present_create_copy (FLAG_CREATE, h, s);
 468 }
 469
 470 void *
 471 acc_copyin (void *h, size_t s)
 472 {
 473   return present_create_copy (FLAG_CREATE | FLAG_COPY, h, s);
 474 }
 475
 476 void *
 477 acc_present_or_create (void *h, size_t s)
 478 {
 479   return present_create_copy (FLAG_PRESENT | FLAG_CREATE, h, s);
 480 }
 481
 482 void *
 483 acc_present_or_copyin (void *h, size_t s)
 484 {
 485   return present_create_copy (FLAG_PRESENT | FLAG_CREATE | FLAG_COPY, h, s);
 486 }
 487
 488 #define FLAG_COPYOUT (1 << 0)
 489
 490 static void
 491 delete_copyout (unsigned f, void *h, size_t s)
 492 {
 493   size_t host_size;
 494   splay_tree_key n;
 495   void *d;
 496   struct goacc_thread *thr = goacc_thread ();
 497   struct gomp_device_descr *acc_dev = thr->dev;
 498
 499   gomp_mutex_lock (&acc_dev->lock);
 500
 501   n = lookup_host (acc_dev, h, s);
 502
 503   /* No need to call lazy open, as the data must already have been
 504      mapped.  */
 505
 506   if (!n)
 507     {
 508       gomp_mutex_unlock (&acc_dev->lock);
 509       gomp_fatal ("[%p,%d] is not mapped", (void *)h, (int)s);
 510     }
 511
 512   d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 513
 514   host_size = n->host_end - n->host_start;
 515
 516   if (n->host_start != (uintptr_t) h || host_size != s)
 517     {
 518       gomp_mutex_unlock (&acc_dev->lock);
 519       gomp_fatal ("[%p,%d] surrounds2 [%p,+%d]",
 520                   (void *) n->host_start, (int) host_size, (void *) h, (int) s);
 521     }
 522
 523   gomp_mutex_unlock (&acc_dev->lock);
 524
 525   if (f & FLAG_COPYOUT)
 526     acc_dev->dev2host_func (acc_dev->target_id, h, d, s);
 527
 528   acc_unmap_data (h);
 529
 530   acc_dev->free_func (acc_dev->target_id, d);
 531 }
 532
 533 void
 534 acc_delete (void *h , size_t s)
 535 {
 536   delete_copyout (0, h, s);
 537 }
 538
 539 void acc_copyout (void *h, size_t s)
 540 {
 541   delete_copyout (FLAG_COPYOUT, h, s);
 542 }
 543
 544 static void
 545 update_dev_host (int is_dev, void *h, size_t s)
 546 {
 547   splay_tree_key n;
 548   void *d;
 549
 550   goacc_lazy_initialize ();
 551
 552   struct goacc_thread *thr = goacc_thread ();
 553   struct gomp_device_descr *acc_dev = thr->dev;
 554
 555   gomp_mutex_lock (&acc_dev->lock);
 556
 557   n = lookup_host (acc_dev, h, s);
 558
 559   if (!n)
 560     {
 561       gomp_mutex_unlock (&acc_dev->lock);
 562       gomp_fatal ("[%p,%d] is not mapped", h, (int)s);
 563     }
 564
 565   d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 566
 567   gomp_mutex_unlock (&acc_dev->lock);
 568
 569   if (is_dev)
 570     acc_dev->host2dev_func (acc_dev->target_id, d, h, s);
 571   else
 572     acc_dev->dev2host_func (acc_dev->target_id, h, d, s);
 573 }
 574
 575 void
 576 acc_update_device (void *h, size_t s)
 577 {
 578   update_dev_host (1, h, s);
 579 }
 580
 581 void
 582 acc_update_self (void *h, size_t s)
 583 {
 584   update_dev_host (0, h, s);
 585 }
 586
 587 void
 588 gomp_acc_insert_pointer (size_t mapnum, void **hostaddrs, size_t *sizes,
 589                          void *kinds)
 590 {
 591   struct target_mem_desc *tgt;
 592   struct goacc_thread *thr = goacc_thread ();
 593   struct gomp_device_descr *acc_dev = thr->dev;
 594
 595   gomp_debug (0, "  %s: prepare mappings\n", __FUNCTION__);
 596   tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs,
 597                        NULL, sizes, kinds, true, GOMP_MAP_VARS_OPENACC);
 598   gomp_debug (0, "  %s: mappings prepared\n", __FUNCTION__);
 599
 600   gomp_mutex_lock (&acc_dev->lock);
 601   tgt->prev = acc_dev->openacc.data_environ;
 602   acc_dev->openacc.data_environ = tgt;
 603   gomp_mutex_unlock (&acc_dev->lock);
 604 }
 605
 606 void
 607 gomp_acc_remove_pointer (void *h, bool force_copyfrom, int async, int mapnum)
 608 {
 609   struct goacc_thread *thr = goacc_thread ();
 610   struct gomp_device_descr *acc_dev = thr->dev;
 611   splay_tree_key n;
 612   struct target_mem_desc *t;
 613   int minrefs = (mapnum == 1) ? 2 : 3;
 614
 615   gomp_mutex_lock (&acc_dev->lock);
 616
 617   n = lookup_host (acc_dev, h, 1);
 618
 619   if (!n)
 620     {
 621       gomp_mutex_unlock (&acc_dev->lock);
 622       gomp_fatal ("%p is not a mapped block", (void *)h);
 623     }
 624
 625   gomp_debug (0, "  %s: restore mappings\n", __FUNCTION__);
 626
 627   t = n->tgt;
 628
 629   struct target_mem_desc *tp;
 630
 631   if (t->refcount == minrefs)
 632     {
 633       /* This is the last reference, so pull the descriptor off the
 634          chain. This avoids gomp_unmap_vars via gomp_unmap_tgt from
 635          freeing the device memory. */
 636       t->tgt_end = 0;
 637       t->to_free = 0;
 638
 639       for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 640            tp = t, t = t->prev)
 641         {
 642           if (n->tgt == t)
 643             {
 644               if (tp)
 645                 tp->prev = t->prev;
 646               else
 647                 acc_dev->openacc.data_environ = t->prev;
 648               break;
 649             }
 650         }
 651     }
 652
 653   if (force_copyfrom)
 654     t->list[0].copy_from = 1;
 655
 656   gomp_mutex_unlock (&acc_dev->lock);
 657
 658   /* If running synchronously, unmap immediately.  */
 659   if (async < acc_async_noval)
 660     gomp_unmap_vars (t, true);
 661   else
 662     {
 663       gomp_copy_from_async (t);
 664       acc_dev->openacc.register_async_cleanup_func (t);
 665     }
 666
 667   gomp_debug (0, "  %s: mappings restored\n", __FUNCTION__);
 668 }