This is the mail archive of the
java-patches@gcc.gnu.org
mailing list for the Java project.
libgcj/9802 UTF-8 surrogate handling and char converter pendingbytes
- From: Mark Wielaard <mark at klomp dot org>
- To: java-patches at gcc dot gnu dot org
- Date: 06 Jun 2003 15:44:19 +0200
- Subject: libgcj/9802 UTF-8 surrogate handling and char converter pendingbytes
Hi,
I have been out of the loop for a while and this is a follow up to a
patch from a few months ago. I didn't really push it back then since I
assumed that 3.3 could be released any moment then.
See http://gcc.gnu.org/ml/java-patches/2003-q1/msg00756.html
I have updated the patch to current CVS and reran all tests.
No regressions and it fixes the Mauve surrogate test.
2002-06-06 James Clark <jjc@jclark dot com>
Fix for PR libgcj/8738:
* gnu/gcj/convert/UnicodeToBytes.java (havePendingBytes): New method.
* gnu/gcj/convert/Output_SJIS.java (havePendingBytes): Likewise.
* gnu/gcj/convert/Output_EUCJIS.java (havePendingBytes): Likewise.
* gnu/gcj/convert/Output_UTF8.java (havePendingBytes): Likewise.
(write): Always decrease avail when count is increased.
* java/lang/natString.cc (getBytes): Check converter havePendingBytes()
and whether output buffer is full before increasing size.
2002-06-06 Mark Wielaard <mark@klomp dot org>
* java/io/PrintStream.java (writeChars(char[],int, int)):
Check converter.havePendingBytes().
(writeChars(String,int,int)): Likewise.
* java/io/OutputStreamWriter.java (writeChars(char[], int, int)):
Check converter.havePendingBytes() and flush buffer when stalled.
OK to check this in now?
It would be a good idea to also fix this on the 3.3 branch.
There is also Tom his patch to remove the iconv hack.
http://gcc.gnu.org/ml/java-patches/2003-q1/msg00603.html
Which should probably also go in, but maybe only into the HEAD.
Cheers,
Mark
Index: gnu/gcj/convert/Output_EUCJIS.java
===================================================================
RCS file: /cvs/gcc/gcc/libjava/gnu/gcj/convert/Output_EUCJIS.java,v
retrieving revision 1.3
diff -u -r1.3 Output_EUCJIS.java
--- gnu/gcj/convert/Output_EUCJIS.java 7 Mar 2000 19:55:24 -0000 1.3
+++ gnu/gcj/convert/Output_EUCJIS.java 6 Jun 2003 13:36:10 -0000
@@ -1,4 +1,4 @@
-/* Copyright (C) 1999 Free Software Foundation
+/* Copyright (C) 1999, 2003 Free Software Foundation
This file is part of libgcj.
@@ -21,6 +21,11 @@
public native int write (char[] inbuffer, int inpos, int inlength);
public native int write (String str, int inpos, int inlength, char[] work);
+
+ public boolean havePendingBytes()
+ {
+ return pending1 >= 0;
+ }
int pending1 = -1;
int pending2;
Index: gnu/gcj/convert/Output_SJIS.java
===================================================================
RCS file: /cvs/gcc/gcc/libjava/gnu/gcj/convert/Output_SJIS.java,v
retrieving revision 1.3
diff -u -r1.3 Output_SJIS.java
--- gnu/gcj/convert/Output_SJIS.java 7 Mar 2000 19:55:24 -0000 1.3
+++ gnu/gcj/convert/Output_SJIS.java 6 Jun 2003 13:36:10 -0000
@@ -1,4 +1,4 @@
-/* Copyright (C) 1999 Free Software Foundation
+/* Copyright (C) 1999, 2003 Free Software Foundation
This file is part of libgcj.
@@ -21,6 +21,11 @@
public native int write (char[] inbuffer, int inpos, int inlength);
public native int write (String str, int inpos, int inlength, char[] work);
+
+ public boolean havePendingBytes()
+ {
+ return pending >= 0;
+ }
int pending = -1;
}
Index: gnu/gcj/convert/Output_UTF8.java
===================================================================
RCS file: /cvs/gcc/gcc/libjava/gnu/gcj/convert/Output_UTF8.java,v
retrieving revision 1.6
diff -u -r1.6 Output_UTF8.java
--- gnu/gcj/convert/Output_UTF8.java 8 Aug 2000 17:35:32 -0000 1.6
+++ gnu/gcj/convert/Output_UTF8.java 6 Jun 2003 13:36:10 -0000
@@ -1,4 +1,4 @@
-/* Copyright (C) 1999, 2000 Free Software Foundation
+/* Copyright (C) 1999, 2000, 2003 Free Software Foundation
This file is part of libgcj.
@@ -104,6 +104,7 @@
{
value = (hi_part - 0xD800) * 0x400 + (ch - 0xDC00) + 0x10000;
buf[count++] = (byte) (0xF0 | (value >> 18));
+ avail--;
bytes_todo = 3;
hi_part = 0;
}
@@ -118,4 +119,10 @@
}
return inpos - start_pos;
}
+
+ public boolean havePendingBytes()
+ {
+ return bytes_todo > 0;
+ }
+
}
Index: gnu/gcj/convert/UnicodeToBytes.java
===================================================================
RCS file: /cvs/gcc/gcc/libjava/gnu/gcj/convert/UnicodeToBytes.java,v
retrieving revision 1.9
diff -u -r1.9 UnicodeToBytes.java
--- gnu/gcj/convert/UnicodeToBytes.java 18 Aug 2001 03:56:01 -0000 1.9
+++ gnu/gcj/convert/UnicodeToBytes.java 6 Jun 2003 13:36:10 -0000
@@ -1,4 +1,4 @@
-/* Copyright (C) 1999, 2000, 2001 Free Software Foundation
+/* Copyright (C) 1999, 2000, 2001, 2003 Free Software Foundation
This file is part of libgcj.
@@ -144,6 +144,17 @@
int srcEnd = inpos + (inlength > work.length ? work.length : inlength);
str.getChars(inpos, srcEnd, work, 0);
return write(work, 0, srcEnd - inpos);
+ }
+
+ /**
+ * Returns true when the converter has consumed some bytes that are
+ * not yet converted to characters because further continuation
+ * bytes are needed. Defaults to false, should be overridden by
+ * decoders that internally store some bytes.
+ */
+ public boolean havePendingBytes()
+ {
+ return false;
}
/** Indicate that the converter is resuable.
Index: java/io/OutputStreamWriter.java
===================================================================
RCS file: /cvs/gcc/gcc/libjava/java/io/OutputStreamWriter.java,v
retrieving revision 1.15
diff -u -r1.15 OutputStreamWriter.java
--- java/io/OutputStreamWriter.java 9 May 2003 07:10:58 -0000 1.15
+++ java/io/OutputStreamWriter.java 6 Jun 2003 13:36:11 -0000
@@ -215,7 +215,7 @@
private void writeChars(char[] buf, int offset, int count)
throws IOException
{
- while (count > 0)
+ while (count > 0 || converter.havePendingBytes())
{
// We must flush if out.count == out.buf.length.
// It is probably a good idea to flush if out.buf is almost full.
@@ -228,6 +228,13 @@
}
converter.setOutput(out.buf, out.count);
int converted = converter.write(buf, offset, count);
+ // Flush if we cannot make progress.
+ if (converted == 0 && out.count == converter.count)
+ {
+ out.flush();
+ if (out.count != 0)
+ throw new IOException("unable to flush output byte buffer");
+ }
offset += converted;
count -= converted;
out.count = converter.count;
Index: java/io/PrintStream.java
===================================================================
RCS file: /cvs/gcc/gcc/libjava/java/io/PrintStream.java,v
retrieving revision 1.17
diff -u -r1.17 PrintStream.java
--- java/io/PrintStream.java 27 May 2003 06:04:28 -0000 1.17
+++ java/io/PrintStream.java 6 Jun 2003 13:36:11 -0000
@@ -1,5 +1,5 @@
/* PrintStream.java -- OutputStream for printing output
- Copyright (C) 1998,2003 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2001, 2003 Free Software Foundation, Inc.
This file is part of GNU Classpath.
@@ -262,7 +262,7 @@
private void writeChars(char[] buf, int offset, int count)
throws IOException
{
- while (count > 0)
+ while (count > 0 || converter.havePendingBytes())
{
converter.setOutput(work_bytes, 0);
int converted = converter.write(buf, offset, count);
@@ -275,7 +275,7 @@
private void writeChars(String str, int offset, int count)
throws IOException
{
- while (count > 0)
+ while (count > 0 || converter.havePendingBytes())
{
converter.setOutput(work_bytes, 0);
int converted = converter.write(str, offset, count, work);
Index: java/lang/natString.cc
===================================================================
RCS file: /cvs/gcc/gcc/libjava/java/lang/natString.cc,v
retrieving revision 1.29
diff -u -r1.29 natString.cc
--- java/lang/natString.cc 28 Apr 2003 18:19:23 -0000 1.29
+++ java/lang/natString.cc 6 Jun 2003 13:36:12 -0000
@@ -602,12 +602,12 @@
jint offset = 0;
gnu::gcj::convert::UnicodeToBytes *converter
= gnu::gcj::convert::UnicodeToBytes::getEncoder(enc);
- while (todo > 0)
+ while (todo > 0 || converter->havePendingBytes())
{
converter->setOutput(buffer, bufpos);
int converted = converter->write(this, offset, todo, NULL);
bufpos = converter->count;
- if (converted == 0)
+ if (converted == 0 && bufpos == converter->count)
{
buflen *= 2;
jbyteArray newbuffer = JvNewByteArray(buflen);
@@ -615,10 +615,10 @@
buffer = newbuffer;
}
else
- {
- offset += converted;
- todo -= converted;
- }
+ bufpos = converter->count;
+
+ offset += converted;
+ todo -= converted;
}
converter->done ();
if (bufpos == buflen)