[patch, fortran] Large transfer support for IO library
Janne Blomqvist
jblomqvi@cc.hut.fi
Wed Sep 28 21:02:00 GMT 2005
Hi again,
here's a slightly updated patch. Compared to the previous one, in
write_block_direct it adds the number of bytes transferred to
ioparm.size after the actual transfer to catch any errors where the
transferred size is different from the intented. It also removes a
comment from reset_stream that was a leftover from my own testing.
--
Janne Blomqvist
-------------- next part --------------
Index: io.h
===================================================================
RCS file: /cvsroot/gcc/gcc/libgfortran/io/io.h,v
retrieving revision 1.32
diff -p -u -r1.32 io.h
--- io.h 26 Sep 2005 20:24:44 -0000 1.32
+++ io.h 28 Sep 2005 20:14:21 -0000
@@ -56,6 +56,8 @@ typedef struct stream
try (*close) (struct stream *);
try (*seek) (struct stream *, gfc_offset);
try (*truncate) (struct stream *);
+ int (*read) (struct stream *, void *, size_t *);
+ int (*write) (struct stream *, const void *, size_t *);
}
stream;
@@ -73,6 +75,8 @@ stream;
#define sseek(s, pos) ((s)->seek)(s, pos)
#define struncate(s) ((s)->truncate)(s)
+#define sread(s, buf, nbytes) ((s)->read)(s, buf, nbytes)
+#define swrite(s, buf, nbytes) ((s)->write)(s, buf, nbytes)
/* Representation of a namelist object in libgfortran
Index: transfer.c
===================================================================
RCS file: /cvsroot/gcc/gcc/libgfortran/io/transfer.c,v
retrieving revision 1.60
diff -p -u -r1.60 transfer.c
--- transfer.c 26 Sep 2005 20:24:45 -0000 1.60
+++ transfer.c 28 Sep 2005 20:14:23 -0000
@@ -286,6 +286,60 @@ read_block (int *length)
}
+/* Reads a block directly into application data space. */
+
+static void
+read_block_direct (void * buf, size_t * nbytes)
+{
+ int *length;
+ void *data;
+ size_t nread;
+
+ if (current_unit->flags.form == FORM_FORMATTED &&
+ current_unit->flags.access == ACCESS_SEQUENTIAL)
+ {
+ length = (int*) nbytes;
+ data = read_sf (length); /* Special case. */
+ memcpy (buf, data, (size_t) *length);
+ return;
+ }
+
+ if (current_unit->bytes_left < *nbytes)
+ {
+ if (current_unit->flags.pad == PAD_NO)
+ {
+ generate_error (ERROR_EOR, NULL); /* Not enough data left. */
+ return;
+ }
+
+ *nbytes = current_unit->bytes_left;
+ }
+
+ current_unit->bytes_left -= *nbytes;
+
+ nread = *nbytes;
+ if (sread (current_unit->s, buf, &nread) != 0)
+ {
+ generate_error (ERROR_OS, NULL);
+ return;
+ }
+
+ if (ioparm.size != NULL)
+ *ioparm.size += (GFC_INTEGER_4) nread;
+
+ if (nread != *nbytes)
+ { /* Short read, this shouldn't happen. */
+ if (current_unit->flags.pad == PAD_YES)
+ {
+ memset (((char *) buf) + nread, ' ', *nbytes - nread);
+ *nbytes = nread;
+ }
+ else
+ generate_error (ERROR_EOR, NULL);
+ }
+}
+
+
/* Function for writing a block of bytes to the current file at the
current position, advancing the file pointer. We are given a length
and return a pointer to a buffer that the caller must (completely)
@@ -312,39 +366,49 @@ write_block (int length)
}
+/* Writes a block directly without necessarily allocating space in a
+ buffer. */
+
+static void
+write_block_direct (void * buf, size_t * nbytes)
+{
+ if (current_unit->bytes_left < *nbytes)
+ generate_error (ERROR_EOR, NULL);
+
+ current_unit->bytes_left -= (gfc_offset) *nbytes;
+
+ if (swrite (current_unit->s, buf, nbytes) != 0)
+ generate_error (ERROR_OS, NULL);
+
+ if (ioparm.size != NULL)
+ *ioparm.size += (GFC_INTEGER_4) *nbytes;
+}
+
+
/* Master function for unformatted reads. */
static void
unformatted_read (bt type, void *dest, int length, size_t nelems)
{
- void *source;
- int w;
+ size_t len;
- length *= nelems;
+ len = length * nelems;
/* Transfer functions get passed the kind of the entity, so we have
to fix this for COMPLEX data which are twice the size of their
kind. */
if (type == BT_COMPLEX)
- length *= 2;
-
- w = length;
- source = read_block (&w);
+ len *= 2;
- if (source != NULL)
- {
- memcpy (dest, source, w);
- if (length != w)
- memset (((char *) dest) + w, ' ', length - w);
- }
+ read_block_direct (dest, &len);
}
+
/* Master function for unformatted writes. */
static void
unformatted_write (bt type, void *source, int length, size_t nelems)
{
- void *dest;
size_t len;
len = length * nelems;
@@ -353,9 +417,7 @@ unformatted_write (bt type, void *source
if (type == BT_COMPLEX)
len *= 2;
- dest = write_block (len);
- if (dest != NULL)
- memcpy (dest, source, len);
+ write_block_direct (source, &len);
}
Index: unix.c
===================================================================
RCS file: /cvsroot/gcc/gcc/libgfortran/io/unix.c,v
retrieving revision 1.40
diff -p -u -r1.40 unix.c
--- unix.c 14 Sep 2005 20:18:18 -0000 1.40
+++ unix.c 28 Sep 2005 20:14:23 -0000
@@ -40,9 +40,6 @@ Boston, MA 02110-1301, USA. */
#include <fcntl.h>
#include <assert.h>
-#ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
-#endif
#include <string.h>
#include <errno.h>
@@ -53,10 +50,6 @@ Boston, MA 02110-1301, USA. */
#define PATH_MAX 1024
#endif
-#ifndef MAP_FAILED
-#define MAP_FAILED ((void *) -1)
-#endif
-
#ifndef PROT_READ
#define PROT_READ 1
#endif
@@ -502,9 +495,15 @@ fd_sfree (unix_stream * s)
}
-static int
+static try
fd_seek (unix_stream * s, gfc_offset offset)
{
+ if (s->physical_offset == offset) /* Are we lucky and avoid syscall? */
+ {
+ s->logical_offset = offset;
+ return SUCCESS;
+ }
+
s->physical_offset = s->logical_offset = offset;
return (lseek (s->fd, offset, SEEK_SET) < 0) ? FAILURE : SUCCESS;
@@ -543,6 +542,173 @@ fd_truncate (unix_stream * s)
}
+inline static void
+reset_stream (unix_stream * s)
+{
+ s->logical_offset = s->physical_offset;
+ if (s->file_length != -1 && s->physical_offset > s->file_length)
+ s->file_length = s->physical_offset;
+}
+
+
+/* Stream read function. Avoids using a buffer for big reads. The
+ interface is like POSIX read(), but the nbytes argument is a
+ pointer; on return it contains the number of bytes written. The
+ function return value is the status indicator (0 for success). */
+
+static int
+fd_read (unix_stream * s, void * buf, size_t * nbytes)
+{
+ void *p;
+ ssize_t bytes_read;
+ int tmp;
+
+ errno = 0;
+
+ if (*nbytes < BUFFER_SIZE && !s->unbuffered)
+ {
+ tmp = *nbytes;
+ p = fd_alloc_r_at (s, &tmp, -1);
+ if (p)
+ {
+ memcpy (buf, p, *nbytes);
+ return 0;
+ }
+ else
+ return errno;
+ }
+ /* If the request if bigger than BUFFER_SIZE we flush the buffers
+ and read directly. */
+ if (fd_flush (s) == FAILURE)
+ {
+ *nbytes = 0;
+ return errno;
+ }
+ if (is_seekable ((stream *) s) && fd_seek (s, s->logical_offset) == FAILURE)
+ {
+ *nbytes = 0;
+ return errno;
+ }
+ /* Requests between SSIZE_MAX and SIZE_MAX are undefined by SUS, in
+ that case we do two writes. */
+ if (*nbytes < SSIZE_MAX)
+ bytes_read = read (s->fd, buf, *nbytes);
+ else
+ bytes_read = read (s->fd, buf, SSIZE_MAX);
+ if (bytes_read < 0)
+ {
+ /* Assume we wrote zero bits, return error. */
+ *nbytes = 0;
+ return errno;
+ }
+ s->physical_offset += bytes_read;
+ reset_stream (s);
+ if (*nbytes < SSIZE_MAX)
+ {
+ *nbytes = bytes_read;
+ return 0;
+ }
+
+ /* Now we handle the remainder of the big write. */
+
+ /* Handle the last write in case it was short and return. */
+ if (bytes_read != SSIZE_MAX)
+ {
+ *nbytes = bytes_read;
+ return 0;
+ }
+ bytes_read = read (s->fd, (char *)buf + SSIZE_MAX, *nbytes - SSIZE_MAX);
+ if (bytes_read < 0)
+ {
+ *nbytes = SSIZE_MAX;
+ return errno;
+ }
+ s->physical_offset += bytes_read;
+ reset_stream (s);
+ *nbytes = SSIZE_MAX + bytes_read;
+ return 0;
+}
+
+
+/* Stream write function. Avoids using a buffer for big writes. The
+ interface is like POSIX write(), but the nbytes argument is a
+ pointer; on return it contains the number of bytes written. The
+ function return value is the status indicator (0 for success). */
+
+static int
+fd_write (unix_stream * s, const void * buf, size_t * nbytes)
+{
+ void *p;
+ ssize_t bytes_written;
+ int tmp;
+
+ errno = 0;
+
+ if (*nbytes < BUFFER_SIZE && !s->unbuffered)
+ {
+ tmp = *nbytes;
+ p = fd_alloc_w_at (s, &tmp, -1);
+ if (p)
+ {
+ memcpy (p, buf, *nbytes);
+ return 0;
+ }
+ else
+ return errno;
+ }
+ /* If the request if bigger than BUFFER_SIZE we flush the buffers
+ and write directly. */
+ if (fd_flush (s) == FAILURE)
+ {
+ *nbytes = 0;
+ return errno;
+ }
+ if (is_seekable ((stream *) s) && fd_seek (s, s->logical_offset) == FAILURE)
+ {
+ *nbytes = 0;
+ return errno;
+ }
+ /* Requests between SSIZE_MAX and SIZE_MAX are undefined by SUS, in
+ that case we do two writes. */
+ if (*nbytes < SSIZE_MAX)
+ bytes_written = write (s->fd, buf, *nbytes);
+ else
+ bytes_written = write (s->fd, buf, SSIZE_MAX);
+ if (bytes_written < 0)
+ {
+ /* Assume we wrote zero bits, return error. */
+ *nbytes = 0;
+ return errno;
+ }
+ s->physical_offset += bytes_written;
+ reset_stream (s);
+ if (*nbytes < SSIZE_MAX)
+ {
+ *nbytes = bytes_written;
+ return 0;
+ }
+
+ /* Now we handle the remainder of the big write. */
+
+ /* Handle the last write in case it was short and return. */
+ if (bytes_written != SSIZE_MAX)
+ {
+ *nbytes = bytes_written;
+ return 0;
+ }
+ bytes_written = write (s->fd, (char *)buf + SSIZE_MAX, *nbytes - SSIZE_MAX);
+ if (bytes_written < 0)
+ {
+ *nbytes = SSIZE_MAX;
+ return errno;
+ }
+ s->physical_offset += bytes_written;
+ reset_stream (s);
+ *nbytes = SSIZE_MAX + bytes_written;
+ return 0;
+}
+
+
static try
fd_close (unix_stream * s)
{
@@ -576,12 +742,15 @@ fd_open (unix_stream * s)
s->st.close = (void *) fd_close;
s->st.seek = (void *) fd_seek;
s->st.truncate = (void *) fd_truncate;
+ s->st.read = (void *) fd_read;
+ s->st.write = (void *) fd_write;
s->buffer = NULL;
}
+
/*********************************************************************
memory stream functions - These are used for internal files
@@ -641,6 +810,58 @@ mem_alloc_w_at (unix_stream * s, int *le
}
+/* Stream read function for internal units. This is not actually used
+ at the moment, as all internal IO is formatted and the formatted IO
+ routines use mem_alloc_r_at function. */
+
+static int
+mem_read (unix_stream * s, void * buf, size_t * nbytes)
+{
+ void *p;
+ int tmp;
+
+ tmp = *nbytes;
+ p = mem_alloc_r_at (s, &tmp, -1);
+ if (p)
+ {
+ memcpy (buf, p, *nbytes);
+ return 0;
+ }
+ else
+ {
+ *nbytes = 0;
+ return errno;
+ }
+}
+
+
+/* Stream write function for internal units. This is not actually used
+ at the moment, as all internal IO is formatted and the formatted IO
+ routines use mem_alloc_w_at. */
+
+static int
+mem_write (unix_stream * s, const void * buf, size_t * nbytes)
+{
+ void *p;
+ int tmp;
+
+ errno = 0;
+
+ tmp = *nbytes;
+ p = mem_alloc_w_at (s, &tmp, -1);
+ if (p)
+ {
+ memcpy (p, buf, *nbytes);
+ return 0;
+ }
+ else
+ {
+ *nbytes = 0;
+ return errno;
+ }
+}
+
+
static int
mem_seek (unix_stream * s, gfc_offset offset)
{
@@ -715,6 +936,8 @@ open_internal (char *base, int length)
s->st.close = (void *) mem_close;
s->st.seek = (void *) mem_seek;
s->st.truncate = (void *) mem_truncate;
+ s->st.read = (void *) mem_read;
+ s->st.write = (void *) mem_write;
return (stream *) s;
}
@@ -1353,9 +1576,8 @@ file_position (stream * s)
int
is_seekable (stream * s)
{
- /* by convention, if file_length == -1, the file is not seekable
- note that a mmapped file is always seekable, an fd_ file may
- or may not be. */
+ /* By convention, if file_length == -1, the file is not
+ seekable. */
return ((unix_stream *) s)->file_length!=-1;
}
More information about the Fortran
mailing list