1 /* CPP Library - charsets
2 Copyright (C) 1998-2014 Free Software Foundation, Inc.
4 Broken out of c-lex.c Apr 2003, adding valid C99 UCN ranges.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
25 /* Character set handling for C-family languages.
27 Terminological note: In what follows, "charset" or "character set"
28 will be taken to mean both an abstract set of characters and an
29 encoding for that set.
31 The C99 standard discusses two character sets: source and execution.
32 The source character set is used for internal processing in translation
33 phases 1 through 4; the execution character set is used thereafter.
34 Both are required by 5.2.1.2p1 to be multibyte encodings, not wide
35 character encodings (see 3.7.2, 3.7.3 for the standardese meanings
36 of these terms). Furthermore, the "basic character set" (listed in
37 5.2.1p3) is to be encoded in each with values one byte wide, and is
38 to appear in the initial shift state.
40 It is not explicitly mentioned, but there is also a "wide execution
41 character set" used to encode wide character constants and wide
42 string literals; this is supposed to be the result of applying the
43 standard library function mbstowcs() to an equivalent narrow string
44 (6.4.5p5). However, the behavior of hexadecimal and octal
45 \-escapes is at odds with this; they are supposed to be translated
46 directly to wchar_t values (6.4.4.4p5,6).
48 The source character set is not necessarily the character set used
49 to encode physical source files on disk; translation phase 1 converts
50 from whatever that encoding is to the source character set.
52 The presence of universal character names in C99 (6.4.3 et seq.)
53 forces the source character set to be isomorphic to ISO 10646,
54 that is, Unicode. There is no such constraint on the execution
55 character set; note also that the conversion from source to
56 execution character set does not occur for identifiers (5.1.1.2p1#5).
58 For convenience of implementation, the source character set's
59 encoding of the basic character set should be identical to the
60 execution character set OF THE HOST SYSTEM's encoding of the basic
61 character set, and it should not be a state-dependent encoding.
63 cpplib uses UTF-8 or UTF-EBCDIC for the source character set,
64 depending on whether the host is based on ASCII or EBCDIC (see
65 respectively Unicode section 2.3/ISO10646 Amendment 2, and Unicode
66 Technical Report #16). With limited exceptions, it relies on the
67 system library's iconv() primitive to do charset conversion
68 (specified in SUSv2). */
71 /* Make certain that the uses of iconv(), iconv_open(), iconv_close()
72 below, which are guarded only by if statements with compile-time
73 constant conditions, do not cause link errors. */
74 #define iconv_open(x, y) (errno = EINVAL, (iconv_t)-1)
75 #define iconv(a,b,c,d,e) (errno = EINVAL, (size_t)-1)
76 #define iconv_close(x) (void)0
80 #if HOST_CHARSET == HOST_CHARSET_ASCII
81 #define SOURCE_CHARSET "UTF-8"
82 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0x7e
83 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
84 #define SOURCE_CHARSET "UTF-EBCDIC"
85 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0xFF
87 #error "Unrecognized basic host character set"
94 /* This structure is used for a resizable string buffer throughout. */
95 /* Don't call it strbuf, as that conflicts with unistd.h on systems
96 such as DYNIX/ptx where unistd.h includes stropts.h. */
104 /* This is enough to hold any string that fits on a single 80-column
105 line, even if iconv quadruples its size (e.g. conversion from
106 ASCII to UTF-32) rounded up to a power of two. */
107 #define OUTBUF_BLOCK_SIZE 256
109 /* Conversions between UTF-8 and UTF-16/32 are implemented by custom
110 logic. This is because a depressing number of systems lack iconv,
111 or have have iconv libraries that do not do these conversions, so
112 we need a fallback implementation for them. To ensure the fallback
113 doesn't break due to neglect, it is used on all systems.
115 UTF-32 encoding is nice and simple: a four-byte binary number,
116 constrained to the range 00000000-7FFFFFFF to avoid questions of
117 signedness. We do have to cope with big- and little-endian
120 UTF-16 encoding uses two-byte binary numbers, again in big- and
121 little-endian variants, for all values in the 00000000-0000FFFF
122 range. Values in the 00010000-0010FFFF range are encoded as pairs
123 of two-byte numbers, called "surrogate pairs": given a number S in
124 this range, it is mapped to a pair (H, L) as follows:
126 H = (S - 0x10000) / 0x400 + 0xD800
127 L = (S - 0x10000) % 0x400 + 0xDC00
129 Two-byte values in the D800...DFFF range are ill-formed except as a
130 component of a surrogate pair. Even if the encoding within a
131 two-byte value is little-endian, the H member of the surrogate pair
134 There is no way to encode values in the 00110000-7FFFFFFF range,
135 which is not currently a problem as there are no assigned code
136 points in that range; however, the author expects that it will
137 eventually become necessary to abandon UTF-16 due to this
138 limitation. Note also that, because of these pairs, UTF-16 does
139 not meet the requirements of the C standard for a wide character
140 encoding (see 3.7.3 and 6.4.4.4p11).
142 UTF-8 encoding looks like this:
144 value range encoded as
145 00000000-0000007F 0xxxxxxx
146 00000080-000007FF 110xxxxx 10xxxxxx
147 00000800-0000FFFF 1110xxxx 10xxxxxx 10xxxxxx
148 00010000-001FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
149 00200000-03FFFFFF 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
150 04000000-7FFFFFFF 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
152 Values in the 0000D800 ... 0000DFFF range (surrogates) are invalid,
153 which means that three-byte sequences ED xx yy, with A0 <= xx <= BF,
154 never occur. Note also that any value that can be encoded by a
155 given row of the table can also be encoded by all successive rows,
156 but this is not done; only the shortest possible encoding for any
157 given value is valid. For instance, the character 07C0 could be
158 encoded as any of DF 80, E0 9F 80, F0 80 9F 80, F8 80 80 9F 80, or
159 FC 80 80 80 9F 80. Only the first is valid.
161 An implementation note: the transformation from UTF-16 to UTF-8, or
162 vice versa, is easiest done by using UTF-32 as an intermediary. */
164 /* Internal primitives which go from an UTF-8 byte stream to native-endian
165 UTF-32 in a cppchar_t, or vice versa; this avoids an extra marshal/unmarshal
166 operation in several places below. */
168 one_utf8_to_cppchar (const uchar
**inbufp
, size_t *inbytesleftp
,
171 static const uchar masks
[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x03, 0x01 };
172 static const uchar patns
[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
175 const uchar
*inbuf
= *inbufp
;
178 if (*inbytesleftp
< 1)
190 /* The number of leading 1-bits in the first byte indicates how many
192 for (nbytes
= 2; nbytes
< 7; nbytes
++)
193 if ((c
& ~masks
[nbytes
-1]) == patns
[nbytes
-1])
198 if (*inbytesleftp
< nbytes
)
201 c
= (c
& masks
[nbytes
-1]);
203 for (i
= 1; i
< nbytes
; i
++)
205 cppchar_t n
= *inbuf
++;
206 if ((n
& 0xC0) != 0x80)
208 c
= ((c
<< 6) + (n
& 0x3F));
211 /* Make sure the shortest possible encoding was used. */
212 if (c
<= 0x7F && nbytes
> 1) return EILSEQ
;
213 if (c
<= 0x7FF && nbytes
> 2) return EILSEQ
;
214 if (c
<= 0xFFFF && nbytes
> 3) return EILSEQ
;
215 if (c
<= 0x1FFFFF && nbytes
> 4) return EILSEQ
;
216 if (c
<= 0x3FFFFFF && nbytes
> 5) return EILSEQ
;
218 /* Make sure the character is valid. */
219 if (c
> 0x7FFFFFFF || (c
>= 0xD800 && c
<= 0xDFFF)) return EILSEQ
;
223 *inbytesleftp
-= nbytes
;
228 one_cppchar_to_utf8 (cppchar_t c
, uchar
**outbufp
, size_t *outbytesleftp
)
230 static const uchar masks
[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
231 static const uchar limits
[6] = { 0x80, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE };
233 uchar buf
[6], *p
= &buf
[6];
234 uchar
*outbuf
= *outbufp
;
243 *--p
= ((c
& 0x3F) | 0x80);
247 while (c
>= 0x3F || (c
& limits
[nbytes
-1]));
248 *--p
= (c
| masks
[nbytes
-1]);
251 if (*outbytesleftp
< nbytes
)
256 *outbytesleftp
-= nbytes
;
261 /* The following four functions transform one character between the two
262 encodings named in the function name. All have the signature
263 int (*)(iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
264 uchar **outbufp, size_t *outbytesleftp)
266 BIGEND must have the value 0 or 1, coerced to (iconv_t); it is
267 interpreted as a boolean indicating whether big-endian or
268 little-endian encoding is to be used for the member of the pair
271 INBUFP, INBYTESLEFTP, OUTBUFP, OUTBYTESLEFTP work exactly as they
274 The return value is either 0 for success, or an errno value for
275 failure, which may be E2BIG (need more space), EILSEQ (ill-formed
276 input sequence), ir EINVAL (incomplete input sequence). */
279 one_utf8_to_utf32 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
280 uchar
**outbufp
, size_t *outbytesleftp
)
286 /* Check for space first, since we know exactly how much we need. */
287 if (*outbytesleftp
< 4)
290 rval
= one_utf8_to_cppchar (inbufp
, inbytesleftp
, &s
);
295 outbuf
[bigend
? 3 : 0] = (s
& 0x000000FF);
296 outbuf
[bigend
? 2 : 1] = (s
& 0x0000FF00) >> 8;
297 outbuf
[bigend
? 1 : 2] = (s
& 0x00FF0000) >> 16;
298 outbuf
[bigend
? 0 : 3] = (s
& 0xFF000000) >> 24;
306 one_utf32_to_utf8 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
307 uchar
**outbufp
, size_t *outbytesleftp
)
313 if (*inbytesleftp
< 4)
318 s
= inbuf
[bigend
? 0 : 3] << 24;
319 s
+= inbuf
[bigend
? 1 : 2] << 16;
320 s
+= inbuf
[bigend
? 2 : 1] << 8;
321 s
+= inbuf
[bigend
? 3 : 0];
323 if (s
>= 0x7FFFFFFF || (s
>= 0xD800 && s
<= 0xDFFF))
326 rval
= one_cppchar_to_utf8 (s
, outbufp
, outbytesleftp
);
336 one_utf8_to_utf16 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
337 uchar
**outbufp
, size_t *outbytesleftp
)
341 const uchar
*save_inbuf
= *inbufp
;
342 size_t save_inbytesleft
= *inbytesleftp
;
343 uchar
*outbuf
= *outbufp
;
345 rval
= one_utf8_to_cppchar (inbufp
, inbytesleftp
, &s
);
351 *inbufp
= save_inbuf
;
352 *inbytesleftp
= save_inbytesleft
;
358 if (*outbytesleftp
< 2)
360 *inbufp
= save_inbuf
;
361 *inbytesleftp
= save_inbytesleft
;
364 outbuf
[bigend
? 1 : 0] = (s
& 0x00FF);
365 outbuf
[bigend
? 0 : 1] = (s
& 0xFF00) >> 8;
375 if (*outbytesleftp
< 4)
377 *inbufp
= save_inbuf
;
378 *inbytesleftp
= save_inbytesleft
;
382 hi
= (s
- 0x10000) / 0x400 + 0xD800;
383 lo
= (s
- 0x10000) % 0x400 + 0xDC00;
385 /* Even if we are little-endian, put the high surrogate first.
386 ??? Matches practice? */
387 outbuf
[bigend
? 1 : 0] = (hi
& 0x00FF);
388 outbuf
[bigend
? 0 : 1] = (hi
& 0xFF00) >> 8;
389 outbuf
[bigend
? 3 : 2] = (lo
& 0x00FF);
390 outbuf
[bigend
? 2 : 3] = (lo
& 0xFF00) >> 8;
399 one_utf16_to_utf8 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
400 uchar
**outbufp
, size_t *outbytesleftp
)
403 const uchar
*inbuf
= *inbufp
;
406 if (*inbytesleftp
< 2)
408 s
= inbuf
[bigend
? 0 : 1] << 8;
409 s
+= inbuf
[bigend
? 1 : 0];
411 /* Low surrogate without immediately preceding high surrogate is invalid. */
412 if (s
>= 0xDC00 && s
<= 0xDFFF)
414 /* High surrogate must have a following low surrogate. */
415 else if (s
>= 0xD800 && s
<= 0xDBFF)
417 cppchar_t hi
= s
, lo
;
418 if (*inbytesleftp
< 4)
421 lo
= inbuf
[bigend
? 2 : 3] << 8;
422 lo
+= inbuf
[bigend
? 3 : 2];
424 if (lo
< 0xDC00 || lo
> 0xDFFF)
427 s
= (hi
- 0xD800) * 0x400 + (lo
- 0xDC00) + 0x10000;
430 rval
= one_cppchar_to_utf8 (s
, outbufp
, outbytesleftp
);
434 /* Success - update the input pointers (one_cppchar_to_utf8 has done
435 the output pointers for us). */
449 /* Helper routine for the next few functions. The 'const' on
450 one_conversion means that we promise not to modify what function is
451 pointed to, which lets the inliner see through it. */
454 conversion_loop (int (*const one_conversion
)(iconv_t
, const uchar
**, size_t *,
456 iconv_t cd
, const uchar
*from
, size_t flen
, struct _cpp_strbuf
*to
)
460 size_t inbytesleft
, outbytesleft
;
465 outbuf
= to
->text
+ to
->len
;
466 outbytesleft
= to
->asize
- to
->len
;
471 rval
= one_conversion (cd
, &inbuf
, &inbytesleft
,
472 &outbuf
, &outbytesleft
);
473 while (inbytesleft
&& !rval
);
475 if (__builtin_expect (inbytesleft
== 0, 1))
477 to
->len
= to
->asize
- outbytesleft
;
486 outbytesleft
+= OUTBUF_BLOCK_SIZE
;
487 to
->asize
+= OUTBUF_BLOCK_SIZE
;
488 to
->text
= XRESIZEVEC (uchar
, to
->text
, to
->asize
);
489 outbuf
= to
->text
+ to
->asize
- outbytesleft
;
494 /* These functions convert entire strings between character sets.
495 They all have the signature
497 bool (*)(iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to);
499 The input string FROM is converted as specified by the function
500 name plus the iconv descriptor CD (which may be fake), and the
501 result appended to TO. On any error, false is returned, otherwise true. */
503 /* These four use the custom conversion code above. */
505 convert_utf8_utf16 (iconv_t cd
, const uchar
*from
, size_t flen
,
506 struct _cpp_strbuf
*to
)
508 return conversion_loop (one_utf8_to_utf16
, cd
, from
, flen
, to
);
512 convert_utf8_utf32 (iconv_t cd
, const uchar
*from
, size_t flen
,
513 struct _cpp_strbuf
*to
)
515 return conversion_loop (one_utf8_to_utf32
, cd
, from
, flen
, to
);
519 convert_utf16_utf8 (iconv_t cd
, const uchar
*from
, size_t flen
,
520 struct _cpp_strbuf
*to
)
522 return conversion_loop (one_utf16_to_utf8
, cd
, from
, flen
, to
);
526 convert_utf32_utf8 (iconv_t cd
, const uchar
*from
, size_t flen
,
527 struct _cpp_strbuf
*to
)
529 return conversion_loop (one_utf32_to_utf8
, cd
, from
, flen
, to
);
532 /* Identity conversion, used when we have no alternative. */
534 convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED
,
535 const uchar
*from
, size_t flen
, struct _cpp_strbuf
*to
)
537 if (to
->len
+ flen
> to
->asize
)
539 to
->asize
= to
->len
+ flen
;
540 to
->text
= XRESIZEVEC (uchar
, to
->text
, to
->asize
);
542 memcpy (to
->text
+ to
->len
, from
, flen
);
547 /* And this one uses the system iconv primitive. It's a little
548 different, since iconv's interface is a little different. */
551 #define CONVERT_ICONV_GROW_BUFFER \
553 outbytesleft += OUTBUF_BLOCK_SIZE; \
554 to->asize += OUTBUF_BLOCK_SIZE; \
555 to->text = XRESIZEVEC (uchar, to->text, to->asize); \
556 outbuf = (char *)to->text + to->asize - outbytesleft; \
560 convert_using_iconv (iconv_t cd
, const uchar
*from
, size_t flen
,
561 struct _cpp_strbuf
*to
)
563 ICONV_CONST
char *inbuf
;
565 size_t inbytesleft
, outbytesleft
;
567 /* Reset conversion descriptor and check that it is valid. */
568 if (iconv (cd
, 0, 0, 0, 0) == (size_t)-1)
571 inbuf
= (ICONV_CONST
char *)from
;
573 outbuf
= (char *)to
->text
+ to
->len
;
574 outbytesleft
= to
->asize
- to
->len
;
578 iconv (cd
, &inbuf
, &inbytesleft
, &outbuf
, &outbytesleft
);
579 if (__builtin_expect (inbytesleft
== 0, 1))
581 /* Close out any shift states, returning to the initial state. */
582 if (iconv (cd
, 0, 0, &outbuf
, &outbytesleft
) == (size_t)-1)
587 CONVERT_ICONV_GROW_BUFFER
;
588 if (iconv (cd
, 0, 0, &outbuf
, &outbytesleft
) == (size_t)-1)
592 to
->len
= to
->asize
- outbytesleft
;
598 CONVERT_ICONV_GROW_BUFFER
;
602 #define convert_using_iconv 0 /* prevent undefined symbol error below */
605 /* Arrange for the above custom conversion logic to be used automatically
606 when conversion between a suitable pair of character sets is requested. */
608 #define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO) \
609 CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO)
617 static const struct conversion conversion_tab
[] = {
618 { "UTF-8/UTF-32LE", convert_utf8_utf32
, (iconv_t
)0 },
619 { "UTF-8/UTF-32BE", convert_utf8_utf32
, (iconv_t
)1 },
620 { "UTF-8/UTF-16LE", convert_utf8_utf16
, (iconv_t
)0 },
621 { "UTF-8/UTF-16BE", convert_utf8_utf16
, (iconv_t
)1 },
622 { "UTF-32LE/UTF-8", convert_utf32_utf8
, (iconv_t
)0 },
623 { "UTF-32BE/UTF-8", convert_utf32_utf8
, (iconv_t
)1 },
624 { "UTF-16LE/UTF-8", convert_utf16_utf8
, (iconv_t
)0 },
625 { "UTF-16BE/UTF-8", convert_utf16_utf8
, (iconv_t
)1 },
628 /* Subroutine of cpp_init_iconv: initialize and return a
629 cset_converter structure for conversion from FROM to TO. If
630 iconv_open() fails, issue an error and return an identity
631 converter. Silently return an identity converter if FROM and TO
633 static struct cset_converter
634 init_iconv_desc (cpp_reader
*pfile
, const char *to
, const char *from
)
636 struct cset_converter ret
;
640 if (!strcasecmp (to
, from
))
642 ret
.func
= convert_no_conversion
;
643 ret
.cd
= (iconv_t
) -1;
648 pair
= (char *) alloca(strlen(to
) + strlen(from
) + 2);
653 for (i
= 0; i
< ARRAY_SIZE (conversion_tab
); i
++)
654 if (!strcasecmp (pair
, conversion_tab
[i
].pair
))
656 ret
.func
= conversion_tab
[i
].func
;
657 ret
.cd
= conversion_tab
[i
].fake_cd
;
662 /* No custom converter - try iconv. */
665 ret
.func
= convert_using_iconv
;
666 ret
.cd
= iconv_open (to
, from
);
669 if (ret
.cd
== (iconv_t
) -1)
672 cpp_error (pfile
, CPP_DL_ERROR
, /* FIXME should be DL_SORRY */
673 "conversion from %s to %s not supported by iconv",
676 cpp_errno (pfile
, CPP_DL_ERROR
, "iconv_open");
678 ret
.func
= convert_no_conversion
;
683 cpp_error (pfile
, CPP_DL_ERROR
, /* FIXME: should be DL_SORRY */
684 "no iconv implementation, cannot convert from %s to %s",
686 ret
.func
= convert_no_conversion
;
687 ret
.cd
= (iconv_t
) -1;
693 /* If charset conversion is requested, initialize iconv(3) descriptors
694 for conversion from the source character set to the execution
695 character sets. If iconv is not present in the C library, and
696 conversion is requested, issue an error. */
699 cpp_init_iconv (cpp_reader
*pfile
)
701 const char *ncset
= CPP_OPTION (pfile
, narrow_charset
);
702 const char *wcset
= CPP_OPTION (pfile
, wide_charset
);
703 const char *default_wcset
;
705 bool be
= CPP_OPTION (pfile
, bytes_big_endian
);
707 if (CPP_OPTION (pfile
, wchar_precision
) >= 32)
708 default_wcset
= be
? "UTF-32BE" : "UTF-32LE";
709 else if (CPP_OPTION (pfile
, wchar_precision
) >= 16)
710 default_wcset
= be
? "UTF-16BE" : "UTF-16LE";
712 /* This effectively means that wide strings are not supported,
713 so don't do any conversion at all. */
714 default_wcset
= SOURCE_CHARSET
;
717 ncset
= SOURCE_CHARSET
;
719 wcset
= default_wcset
;
721 pfile
->narrow_cset_desc
= init_iconv_desc (pfile
, ncset
, SOURCE_CHARSET
);
722 pfile
->narrow_cset_desc
.width
= CPP_OPTION (pfile
, char_precision
);
723 pfile
->utf8_cset_desc
= init_iconv_desc (pfile
, "UTF-8", SOURCE_CHARSET
);
724 pfile
->utf8_cset_desc
.width
= CPP_OPTION (pfile
, char_precision
);
725 pfile
->char16_cset_desc
= init_iconv_desc (pfile
,
726 be
? "UTF-16BE" : "UTF-16LE",
728 pfile
->char16_cset_desc
.width
= 16;
729 pfile
->char32_cset_desc
= init_iconv_desc (pfile
,
730 be
? "UTF-32BE" : "UTF-32LE",
732 pfile
->char32_cset_desc
.width
= 32;
733 pfile
->wide_cset_desc
= init_iconv_desc (pfile
, wcset
, SOURCE_CHARSET
);
734 pfile
->wide_cset_desc
.width
= CPP_OPTION (pfile
, wchar_precision
);
737 /* Destroy iconv(3) descriptors set up by cpp_init_iconv, if necessary. */
739 _cpp_destroy_iconv (cpp_reader
*pfile
)
743 if (pfile
->narrow_cset_desc
.func
== convert_using_iconv
)
744 iconv_close (pfile
->narrow_cset_desc
.cd
);
745 if (pfile
->utf8_cset_desc
.func
== convert_using_iconv
)
746 iconv_close (pfile
->utf8_cset_desc
.cd
);
747 if (pfile
->char16_cset_desc
.func
== convert_using_iconv
)
748 iconv_close (pfile
->char16_cset_desc
.cd
);
749 if (pfile
->char32_cset_desc
.func
== convert_using_iconv
)
750 iconv_close (pfile
->char32_cset_desc
.cd
);
751 if (pfile
->wide_cset_desc
.func
== convert_using_iconv
)
752 iconv_close (pfile
->wide_cset_desc
.cd
);
756 /* Utility routine for use by a full compiler. C is a character taken
757 from the *basic* source character set, encoded in the host's
758 execution encoding. Convert it to (the target's) execution
759 encoding, and return that value.
761 Issues an internal error if C's representation in the narrow
762 execution character set fails to be a single-byte value (C99
763 5.2.1p3: "The representation of each member of the source and
764 execution character sets shall fit in a byte.") May also issue an
765 internal error if C fails to be a member of the basic source
766 character set (testing this exactly is too hard, especially when
767 the host character set is EBCDIC). */
769 cpp_host_to_exec_charset (cpp_reader
*pfile
, cppchar_t c
)
772 struct _cpp_strbuf tbuf
;
774 /* This test is merely an approximation, but it suffices to catch
775 the most important thing, which is that we don't get handed a
776 character outside the unibyte range of the host character set. */
777 if (c
> LAST_POSSIBLY_BASIC_SOURCE_CHAR
)
779 cpp_error (pfile
, CPP_DL_ICE
,
780 "character 0x%lx is not in the basic source character set\n",
785 /* Being a character in the unibyte range of the host character set,
786 we can safely splat it into a one-byte buffer and trust that that
787 is a well-formed string. */
790 /* This should never need to reallocate, but just in case... */
792 tbuf
.text
= XNEWVEC (uchar
, tbuf
.asize
);
795 if (!APPLY_CONVERSION (pfile
->narrow_cset_desc
, sbuf
, 1, &tbuf
))
797 cpp_errno (pfile
, CPP_DL_ICE
, "converting to execution character set");
802 cpp_error (pfile
, CPP_DL_ICE
,
803 "character 0x%lx is not unibyte in execution character set",
814 /* Utility routine that computes a mask of the form 0000...111... with
817 width_to_mask (size_t width
)
819 width
= MIN (width
, BITS_PER_CPPCHAR_T
);
820 if (width
>= CHAR_BIT
* sizeof (size_t))
823 return ((size_t) 1 << width
) - 1;
826 /* A large table of unicode character information. */
828 /* Valid in a C99 identifier? */
830 /* Valid in a C99 identifier, but not as the first character? */
832 /* Valid in a C++ identifier? */
834 /* Valid in a C11/C++11 identifier? */
836 /* Valid in a C11/C++11 identifier, but not as the first character? */
838 /* NFC representation is not valid in an identifier? */
840 /* Might be valid NFC form? */
842 /* Might be valid NFKC form? */
844 /* Certain preceding characters might make it not valid NFC/NKFC form? */
849 /* Bitmap of flags above. */
850 unsigned short flags
;
851 /* Combining class of the character. */
852 unsigned char combine
;
853 /* Last character in the range described by this entry. */
858 /* Returns 1 if C is valid in an identifier, 2 if C is valid except at
859 the start of an identifier, and 0 if C is not valid in an
860 identifier. We assume C has already gone through the checks of
861 _cpp_valid_ucn. Also update NST for C if returning nonzero. The
862 algorithm is a simple binary search on the table defined in
866 ucn_valid_in_identifier (cpp_reader
*pfile
, cppchar_t c
,
867 struct normalize_state
*nst
)
870 unsigned short valid_flags
, invalid_start_flags
;
876 mx
= ARRAY_SIZE (ucnranges
) - 1;
880 if (c
<= ucnranges
[md
].end
)
886 /* When -pedantic, we require the character to have been listed by
887 the standard for the current language. Otherwise, we accept the
888 union of the acceptable sets for all supported language versions. */
889 valid_flags
= C99
| CXX
| C11
;
890 if (CPP_PEDANTIC (pfile
))
892 if (CPP_OPTION (pfile
, c11_identifiers
))
894 else if (CPP_OPTION (pfile
, c99
))
896 else if (CPP_OPTION (pfile
, cplusplus
))
899 if (! (ucnranges
[mn
].flags
& valid_flags
))
901 if (CPP_OPTION (pfile
, c11_identifiers
))
902 invalid_start_flags
= N11
;
903 else if (CPP_OPTION (pfile
, c99
))
904 invalid_start_flags
= N99
;
906 invalid_start_flags
= 0;
909 if (ucnranges
[mn
].combine
!= 0 && ucnranges
[mn
].combine
< nst
->prev_class
)
910 nst
->level
= normalized_none
;
911 else if (ucnranges
[mn
].flags
& CTX
)
914 cppchar_t p
= nst
->previous
;
916 /* For Hangul, characters in the range AC00-D7A3 are NFC/NFKC,
917 and are combined algorithmically from a sequence of the form
918 1100-1112 1161-1175 11A8-11C2
919 (if the third is not present, it is treated as 11A7, which is not
920 really a valid character).
921 Unfortunately, C99 allows (only) the NFC form, but C++ allows
922 only the combining characters. */
923 if (c
>= 0x1161 && c
<= 0x1175)
924 safe
= p
< 0x1100 || p
> 0x1112;
925 else if (c
>= 0x11A8 && c
<= 0x11C2)
926 safe
= (p
< 0xAC00 || p
> 0xD7A3 || (p
- 0xAC00) % 28 != 0);
928 safe
= check_nfc (pfile
, c
, p
);
931 if ((c
>= 0x1161 && c
<= 0x1175) || (c
>= 0x11A8 && c
<= 0x11C2))
932 nst
->level
= MAX (nst
->level
, normalized_identifier_C
);
934 nst
->level
= normalized_none
;
937 else if (ucnranges
[mn
].flags
& NKC
)
939 else if (ucnranges
[mn
].flags
& NFC
)
940 nst
->level
= MAX (nst
->level
, normalized_C
);
941 else if (ucnranges
[mn
].flags
& CID
)
942 nst
->level
= MAX (nst
->level
, normalized_identifier_C
);
944 nst
->level
= normalized_none
;
945 if (ucnranges
[mn
].combine
== 0)
947 nst
->prev_class
= ucnranges
[mn
].combine
;
949 /* In C99, UCN digits may not begin identifiers. In C11 and C++11,
950 UCN combining characters may not begin identifiers. */
951 if (ucnranges
[mn
].flags
& invalid_start_flags
)
957 /* [lex.charset]: The character designated by the universal character
958 name \UNNNNNNNN is that character whose character short name in
959 ISO/IEC 10646 is NNNNNNNN; the character designated by the
960 universal character name \uNNNN is that character whose character
961 short name in ISO/IEC 10646 is 0000NNNN. If the hexadecimal value
962 for a universal character name corresponds to a surrogate code point
963 (in the range 0xD800-0xDFFF, inclusive), the program is ill-formed.
964 Additionally, if the hexadecimal value for a universal-character-name
965 outside a character or string literal corresponds to a control character
966 (in either of the ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a
967 character in the basic source character set, the program is ill-formed.
969 C99 6.4.3: A universal character name shall not specify a character
970 whose short identifier is less than 00A0 other than 0024 ($), 0040 (@),
971 or 0060 (`), nor one in the range D800 through DFFF inclusive.
973 *PSTR must be preceded by "\u" or "\U"; it is assumed that the
974 buffer end is delimited by a non-hex digit. Returns zero if the
975 UCN has not been consumed.
977 Otherwise the nonzero value of the UCN, whether valid or invalid,
978 is returned. Diagnostics are emitted for invalid values. PSTR
979 is updated to point one beyond the UCN, or to the syntactically
982 IDENTIFIER_POS is 0 when not in an identifier, 1 for the start of
983 an identifier, or 2 otherwise. */
986 _cpp_valid_ucn (cpp_reader
*pfile
, const uchar
**pstr
,
987 const uchar
*limit
, int identifier_pos
,
988 struct normalize_state
*nst
)
992 const uchar
*str
= *pstr
;
993 const uchar
*base
= str
- 2;
995 if (!CPP_OPTION (pfile
, cplusplus
) && !CPP_OPTION (pfile
, c99
))
996 cpp_error (pfile
, CPP_DL_WARNING
,
997 "universal character names are only valid in C++ and C99");
998 else if (CPP_OPTION (pfile
, cpp_warn_c90_c99_compat
)
999 && !CPP_OPTION (pfile
, cplusplus
))
1000 cpp_error (pfile
, CPP_DL_WARNING
,
1001 "C99's universal character names are incompatible with C90");
1002 else if (CPP_WTRADITIONAL (pfile
) && identifier_pos
== 0)
1003 cpp_warning (pfile
, CPP_W_TRADITIONAL
,
1004 "the meaning of '\\%c' is different in traditional C",
1009 else if (str
[-1] == 'U')
1013 cpp_error (pfile
, CPP_DL_ICE
, "In _cpp_valid_ucn but not a UCN");
1024 result
= (result
<< 4) + hex_value (c
);
1026 while (--length
&& str
< limit
);
1028 /* Partial UCNs are not valid in strings, but decompose into
1029 multiple tokens in identifiers, so we can't give a helpful
1030 error message in that case. */
1031 if (length
&& identifier_pos
)
1037 cpp_error (pfile
, CPP_DL_ERROR
,
1038 "incomplete universal character name %.*s",
1039 (int) (str
- base
), base
);
1042 /* The C99 standard permits $, @ and ` to be specified as UCNs. We use
1043 hex escapes so that this also works with EBCDIC hosts.
1044 C++0x permits everything below 0xa0 within literals;
1045 ucn_valid_in_identifier will complain about identifiers. */
1046 else if ((result
< 0xa0
1047 && !CPP_OPTION (pfile
, cplusplus
)
1048 && (result
!= 0x24 && result
!= 0x40 && result
!= 0x60))
1049 || (result
& 0x80000000)
1050 || (result
>= 0xD800 && result
<= 0xDFFF))
1052 cpp_error (pfile
, CPP_DL_ERROR
,
1053 "%.*s is not a valid universal character",
1054 (int) (str
- base
), base
);
1057 else if (identifier_pos
&& result
== 0x24
1058 && CPP_OPTION (pfile
, dollars_in_ident
))
1060 if (CPP_OPTION (pfile
, warn_dollars
) && !pfile
->state
.skipping
)
1062 CPP_OPTION (pfile
, warn_dollars
) = 0;
1063 cpp_error (pfile
, CPP_DL_PEDWARN
, "'$' in identifier or number");
1065 NORMALIZE_STATE_UPDATE_IDNUM (nst
, result
);
1067 else if (identifier_pos
)
1069 int validity
= ucn_valid_in_identifier (pfile
, result
, nst
);
1072 cpp_error (pfile
, CPP_DL_ERROR
,
1073 "universal character %.*s is not valid in an identifier",
1074 (int) (str
- base
), base
);
1075 else if (validity
== 2 && identifier_pos
== 1)
1076 cpp_error (pfile
, CPP_DL_ERROR
,
1077 "universal character %.*s is not valid at the start of an identifier",
1078 (int) (str
- base
), base
);
1087 /* Convert an UCN, pointed to by FROM, to UTF-8 encoding, then translate
1088 it to the execution character set and write the result into TBUF.
1089 An advanced pointer is returned. Issues all relevant diagnostics. */
1090 static const uchar
*
1091 convert_ucn (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
1092 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
)
1097 size_t bytesleft
= 6;
1099 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
1101 from
++; /* Skip u/U. */
1102 ucn
= _cpp_valid_ucn (pfile
, &from
, limit
, 0, &nst
);
1104 rval
= one_cppchar_to_utf8 (ucn
, &bufp
, &bytesleft
);
1108 cpp_errno (pfile
, CPP_DL_ERROR
,
1109 "converting UCN to source character set");
1111 else if (!APPLY_CONVERSION (cvt
, buf
, 6 - bytesleft
, tbuf
))
1112 cpp_errno (pfile
, CPP_DL_ERROR
,
1113 "converting UCN to execution character set");
1118 /* Subroutine of convert_hex and convert_oct. N is the representation
1119 in the execution character set of a numeric escape; write it into the
1120 string buffer TBUF and update the end-of-string pointer therein. WIDE
1121 is true if it's a wide string that's being assembled in TBUF. This
1122 function issues no diagnostics and never fails. */
1124 emit_numeric_escape (cpp_reader
*pfile
, cppchar_t n
,
1125 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
)
1127 size_t width
= cvt
.width
;
1129 if (width
!= CPP_OPTION (pfile
, char_precision
))
1131 /* We have to render this into the target byte order, which may not
1132 be our byte order. */
1133 bool bigend
= CPP_OPTION (pfile
, bytes_big_endian
);
1134 size_t cwidth
= CPP_OPTION (pfile
, char_precision
);
1135 size_t cmask
= width_to_mask (cwidth
);
1136 size_t nbwc
= width
/ cwidth
;
1138 size_t off
= tbuf
->len
;
1141 if (tbuf
->len
+ nbwc
> tbuf
->asize
)
1143 tbuf
->asize
+= OUTBUF_BLOCK_SIZE
;
1144 tbuf
->text
= XRESIZEVEC (uchar
, tbuf
->text
, tbuf
->asize
);
1147 for (i
= 0; i
< nbwc
; i
++)
1151 tbuf
->text
[off
+ (bigend
? nbwc
- i
- 1 : i
)] = c
;
1157 /* Note: this code does not handle the case where the target
1158 and host have a different number of bits in a byte. */
1159 if (tbuf
->len
+ 1 > tbuf
->asize
)
1161 tbuf
->asize
+= OUTBUF_BLOCK_SIZE
;
1162 tbuf
->text
= XRESIZEVEC (uchar
, tbuf
->text
, tbuf
->asize
);
1164 tbuf
->text
[tbuf
->len
++] = n
;
1168 /* Convert a hexadecimal escape, pointed to by FROM, to the execution
1169 character set and write it into the string buffer TBUF. Returns an
1170 advanced pointer, and issues diagnostics as necessary.
1171 No character set translation occurs; this routine always produces the
1172 execution-set character with numeric value equal to the given hex
1173 number. You can, e.g. generate surrogate pairs this way. */
1174 static const uchar
*
1175 convert_hex (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
1176 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
)
1178 cppchar_t c
, n
= 0, overflow
= 0;
1179 int digits_found
= 0;
1180 size_t width
= cvt
.width
;
1181 size_t mask
= width_to_mask (width
);
1183 if (CPP_WTRADITIONAL (pfile
))
1184 cpp_warning (pfile
, CPP_W_TRADITIONAL
,
1185 "the meaning of '\\x' is different in traditional C");
1187 from
++; /* Skip 'x'. */
1188 while (from
< limit
)
1194 overflow
|= n
^ (n
<< 4 >> 4);
1195 n
= (n
<< 4) + hex_value (c
);
1201 cpp_error (pfile
, CPP_DL_ERROR
,
1202 "\\x used with no following hex digits");
1206 if (overflow
| (n
!= (n
& mask
)))
1208 cpp_error (pfile
, CPP_DL_PEDWARN
,
1209 "hex escape sequence out of range");
1213 emit_numeric_escape (pfile
, n
, tbuf
, cvt
);
1218 /* Convert an octal escape, pointed to by FROM, to the execution
1219 character set and write it into the string buffer TBUF. Returns an
1220 advanced pointer, and issues diagnostics as necessary.
1221 No character set translation occurs; this routine always produces the
1222 execution-set character with numeric value equal to the given octal
1224 static const uchar
*
1225 convert_oct (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
1226 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
)
1230 size_t width
= cvt
.width
;
1231 size_t mask
= width_to_mask (width
);
1232 bool overflow
= false;
1234 while (from
< limit
&& count
++ < 3)
1237 if (c
< '0' || c
> '7')
1240 overflow
|= n
^ (n
<< 3 >> 3);
1241 n
= (n
<< 3) + c
- '0';
1244 if (n
!= (n
& mask
))
1246 cpp_error (pfile
, CPP_DL_PEDWARN
,
1247 "octal escape sequence out of range");
1251 emit_numeric_escape (pfile
, n
, tbuf
, cvt
);
1256 /* Convert an escape sequence (pointed to by FROM) to its value on
1257 the target, and to the execution character set. Do not scan past
1258 LIMIT. Write the converted value into TBUF. Returns an advanced
1259 pointer. Handles all relevant diagnostics. */
1260 static const uchar
*
1261 convert_escape (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
1262 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
)
1264 /* Values of \a \b \e \f \n \r \t \v respectively. */
1265 #if HOST_CHARSET == HOST_CHARSET_ASCII
1266 static const uchar charconsts
[] = { 7, 8, 27, 12, 10, 13, 9, 11 };
1267 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
1268 static const uchar charconsts
[] = { 47, 22, 39, 12, 21, 13, 5, 11 };
1270 #error "unknown host character set"
1278 /* UCNs, hex escapes, and octal escapes are processed separately. */
1280 return convert_ucn (pfile
, from
, limit
, tbuf
, cvt
);
1283 return convert_hex (pfile
, from
, limit
, tbuf
, cvt
);
1286 case '0': case '1': case '2': case '3':
1287 case '4': case '5': case '6': case '7':
1288 return convert_oct (pfile
, from
, limit
, tbuf
, cvt
);
1290 /* Various letter escapes. Get the appropriate host-charset
1292 case '\\': case '\'': case '"': case '?': break;
1294 case '(': case '{': case '[': case '%':
1295 /* '\(', etc, can be used at the beginning of a line in a long
1296 string split onto multiple lines with \-newline, to prevent
1297 Emacs or other text editors from getting confused. '\%' can
1298 be used to prevent SCCS from mangling printf format strings. */
1299 if (CPP_PEDANTIC (pfile
))
1303 case 'b': c
= charconsts
[1]; break;
1304 case 'f': c
= charconsts
[3]; break;
1305 case 'n': c
= charconsts
[4]; break;
1306 case 'r': c
= charconsts
[5]; break;
1307 case 't': c
= charconsts
[6]; break;
1308 case 'v': c
= charconsts
[7]; break;
1311 if (CPP_WTRADITIONAL (pfile
))
1312 cpp_warning (pfile
, CPP_W_TRADITIONAL
,
1313 "the meaning of '\\a' is different in traditional C");
1318 if (CPP_PEDANTIC (pfile
))
1319 cpp_error (pfile
, CPP_DL_PEDWARN
,
1320 "non-ISO-standard escape sequence, '\\%c'", (int) c
);
1327 cpp_error (pfile
, CPP_DL_PEDWARN
,
1328 "unknown escape sequence: '\\%c'", (int) c
);
1331 /* diagnostic.c does not support "%03o". When it does, this
1332 code can use %03o directly in the diagnostic again. */
1334 sprintf(buf
, "%03o", (int) c
);
1335 cpp_error (pfile
, CPP_DL_PEDWARN
,
1336 "unknown escape sequence: '\\%s'", buf
);
1340 /* Now convert what we have to the execution character set. */
1341 if (!APPLY_CONVERSION (cvt
, &c
, 1, tbuf
))
1342 cpp_errno (pfile
, CPP_DL_ERROR
,
1343 "converting escape sequence to execution character set");
1348 /* TYPE is a token type. The return value is the conversion needed to
1349 convert from source to execution character set for the given type. */
1350 static struct cset_converter
1351 converter_for_type (cpp_reader
*pfile
, enum cpp_ttype type
)
1356 return pfile
->narrow_cset_desc
;
1357 case CPP_UTF8STRING
:
1358 return pfile
->utf8_cset_desc
;
1361 return pfile
->char16_cset_desc
;
1364 return pfile
->char32_cset_desc
;
1367 return pfile
->wide_cset_desc
;
1371 /* FROM is an array of cpp_string structures of length COUNT. These
1372 are to be converted from the source to the execution character set,
1373 escape sequences translated, and finally all are to be
1374 concatenated. WIDE indicates whether or not to produce a wide
1375 string. The result is written into TO. Returns true for success,
1376 false for failure. */
1378 cpp_interpret_string (cpp_reader
*pfile
, const cpp_string
*from
, size_t count
,
1379 cpp_string
*to
, enum cpp_ttype type
)
1381 struct _cpp_strbuf tbuf
;
1382 const uchar
*p
, *base
, *limit
;
1384 struct cset_converter cvt
= converter_for_type (pfile
, type
);
1386 tbuf
.asize
= MAX (OUTBUF_BLOCK_SIZE
, from
->len
);
1387 tbuf
.text
= XNEWVEC (uchar
, tbuf
.asize
);
1390 for (i
= 0; i
< count
; i
++)
1398 else if (*p
== 'L' || *p
== 'U') p
++;
1401 const uchar
*prefix
;
1403 /* Skip over 'R"'. */
1409 limit
= from
[i
].text
+ from
[i
].len
;
1410 if (limit
>= p
+ (p
- prefix
) + 1)
1411 limit
-= (p
- prefix
) + 1;
1413 /* Raw strings are all normal characters; these can be fed
1414 directly to convert_cset. */
1415 if (!APPLY_CONVERSION (cvt
, p
, limit
- p
, &tbuf
))
1421 p
++; /* Skip leading quote. */
1422 limit
= from
[i
].text
+ from
[i
].len
- 1; /* Skip trailing quote. */
1427 while (p
< limit
&& *p
!= '\\')
1431 /* We have a run of normal characters; these can be fed
1432 directly to convert_cset. */
1433 if (!APPLY_CONVERSION (cvt
, base
, p
- base
, &tbuf
))
1439 p
= convert_escape (pfile
, p
+ 1, limit
, &tbuf
, cvt
);
1442 /* NUL-terminate the 'to' buffer and translate it to a cpp_string
1444 emit_numeric_escape (pfile
, 0, &tbuf
, cvt
);
1445 tbuf
.text
= XRESIZEVEC (uchar
, tbuf
.text
, tbuf
.len
);
1446 to
->text
= tbuf
.text
;
1451 cpp_errno (pfile
, CPP_DL_ERROR
, "converting to execution character set");
1456 /* Subroutine of do_line and do_linemarker. Convert escape sequences
1457 in a string, but do not perform character set conversion. */
1459 cpp_interpret_string_notranslate (cpp_reader
*pfile
, const cpp_string
*from
,
1460 size_t count
, cpp_string
*to
,
1461 enum cpp_ttype type ATTRIBUTE_UNUSED
)
1463 struct cset_converter save_narrow_cset_desc
= pfile
->narrow_cset_desc
;
1466 pfile
->narrow_cset_desc
.func
= convert_no_conversion
;
1467 pfile
->narrow_cset_desc
.cd
= (iconv_t
) -1;
1468 pfile
->narrow_cset_desc
.width
= CPP_OPTION (pfile
, char_precision
);
1470 retval
= cpp_interpret_string (pfile
, from
, count
, to
, CPP_STRING
);
1472 pfile
->narrow_cset_desc
= save_narrow_cset_desc
;
1477 /* Subroutine of cpp_interpret_charconst which performs the conversion
1478 to a number, for narrow strings. STR is the string structure returned
1479 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
1480 cpp_interpret_charconst. */
1482 narrow_str_to_charconst (cpp_reader
*pfile
, cpp_string str
,
1483 unsigned int *pchars_seen
, int *unsignedp
)
1485 size_t width
= CPP_OPTION (pfile
, char_precision
);
1486 size_t max_chars
= CPP_OPTION (pfile
, int_precision
) / width
;
1487 size_t mask
= width_to_mask (width
);
1489 cppchar_t result
, c
;
1492 /* The value of a multi-character character constant, or a
1493 single-character character constant whose representation in the
1494 execution character set is more than one byte long, is
1495 implementation defined. This implementation defines it to be the
1496 number formed by interpreting the byte sequence in memory as a
1497 big-endian binary number. If overflow occurs, the high bytes are
1498 lost, and a warning is issued.
1500 We don't want to process the NUL terminator handed back by
1501 cpp_interpret_string. */
1503 for (i
= 0; i
< str
.len
- 1; i
++)
1505 c
= str
.text
[i
] & mask
;
1506 if (width
< BITS_PER_CPPCHAR_T
)
1507 result
= (result
<< width
) | c
;
1515 cpp_error (pfile
, CPP_DL_WARNING
,
1516 "character constant too long for its type");
1518 else if (i
> 1 && CPP_OPTION (pfile
, warn_multichar
))
1519 cpp_warning (pfile
, CPP_W_MULTICHAR
, "multi-character character constant");
1521 /* Multichar constants are of type int and therefore signed. */
1525 unsigned_p
= CPP_OPTION (pfile
, unsigned_char
);
1527 /* Truncate the constant to its natural width, and simultaneously
1528 sign- or zero-extend to the full width of cppchar_t.
1529 For single-character constants, the value is WIDTH bits wide.
1530 For multi-character constants, the value is INT_PRECISION bits wide. */
1532 width
= CPP_OPTION (pfile
, int_precision
);
1533 if (width
< BITS_PER_CPPCHAR_T
)
1535 mask
= ((cppchar_t
) 1 << width
) - 1;
1536 if (unsigned_p
|| !(result
& (1 << (width
- 1))))
1542 *unsignedp
= unsigned_p
;
1546 /* Subroutine of cpp_interpret_charconst which performs the conversion
1547 to a number, for wide strings. STR is the string structure returned
1548 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
1549 cpp_interpret_charconst. TYPE is the token type. */
1551 wide_str_to_charconst (cpp_reader
*pfile
, cpp_string str
,
1552 unsigned int *pchars_seen
, int *unsignedp
,
1553 enum cpp_ttype type
)
1555 bool bigend
= CPP_OPTION (pfile
, bytes_big_endian
);
1556 size_t width
= converter_for_type (pfile
, type
).width
;
1557 size_t cwidth
= CPP_OPTION (pfile
, char_precision
);
1558 size_t mask
= width_to_mask (width
);
1559 size_t cmask
= width_to_mask (cwidth
);
1560 size_t nbwc
= width
/ cwidth
;
1562 cppchar_t result
= 0, c
;
1564 /* This is finicky because the string is in the target's byte order,
1565 which may not be our byte order. Only the last character, ignoring
1566 the NUL terminator, is relevant. */
1567 off
= str
.len
- (nbwc
* 2);
1569 for (i
= 0; i
< nbwc
; i
++)
1571 c
= bigend
? str
.text
[off
+ i
] : str
.text
[off
+ nbwc
- i
- 1];
1572 result
= (result
<< cwidth
) | (c
& cmask
);
1575 /* Wide character constants have type wchar_t, and a single
1576 character exactly fills a wchar_t, so a multi-character wide
1577 character constant is guaranteed to overflow. */
1578 if (str
.len
> nbwc
* 2)
1579 cpp_error (pfile
, CPP_DL_WARNING
,
1580 "character constant too long for its type");
1582 /* Truncate the constant to its natural width, and simultaneously
1583 sign- or zero-extend to the full width of cppchar_t. */
1584 if (width
< BITS_PER_CPPCHAR_T
)
1586 if (type
== CPP_CHAR16
|| type
== CPP_CHAR32
1587 || CPP_OPTION (pfile
, unsigned_wchar
)
1588 || !(result
& (1 << (width
- 1))))
1594 if (type
== CPP_CHAR16
|| type
== CPP_CHAR32
1595 || CPP_OPTION (pfile
, unsigned_wchar
))
1604 /* Interpret a (possibly wide) character constant in TOKEN.
1605 PCHARS_SEEN points to a variable that is filled in with the number
1606 of characters seen, and UNSIGNEDP to a variable that indicates
1607 whether the result has signed type. */
1609 cpp_interpret_charconst (cpp_reader
*pfile
, const cpp_token
*token
,
1610 unsigned int *pchars_seen
, int *unsignedp
)
1612 cpp_string str
= { 0, 0 };
1613 bool wide
= (token
->type
!= CPP_CHAR
);
1616 /* an empty constant will appear as L'', u'', U'' or '' */
1617 if (token
->val
.str
.len
== (size_t) (2 + wide
))
1619 cpp_error (pfile
, CPP_DL_ERROR
, "empty character constant");
1622 else if (!cpp_interpret_string (pfile
, &token
->val
.str
, 1, &str
, token
->type
))
1626 result
= wide_str_to_charconst (pfile
, str
, pchars_seen
, unsignedp
,
1629 result
= narrow_str_to_charconst (pfile
, str
, pchars_seen
, unsignedp
);
1631 if (str
.text
!= token
->val
.str
.text
)
1632 free ((void *)str
.text
);
1637 /* Convert an identifier denoted by ID and LEN, which might contain
1638 UCN escapes, to the source character set, either UTF-8 or
1639 UTF-EBCDIC. Assumes that the identifier is actually a valid identifier. */
1641 _cpp_interpret_identifier (cpp_reader
*pfile
, const uchar
*id
, size_t len
)
1643 /* It turns out that a UCN escape always turns into fewer characters
1644 than the escape itself, so we can allocate a temporary in advance. */
1645 uchar
* buf
= (uchar
*) alloca (len
+ 1);
1649 for (idp
= 0; idp
< len
; idp
++)
1650 if (id
[idp
] != '\\')
1654 unsigned length
= id
[idp
+1] == 'u' ? 4 : 8;
1655 cppchar_t value
= 0;
1656 size_t bufleft
= len
- (bufp
- buf
);
1660 while (length
&& idp
< len
&& ISXDIGIT (id
[idp
]))
1662 value
= (value
<< 4) + hex_value (id
[idp
]);
1668 /* Special case for EBCDIC: if the identifier contains
1669 a '$' specified using a UCN, translate it to EBCDIC. */
1676 rval
= one_cppchar_to_utf8 (value
, &bufp
, &bufleft
);
1680 cpp_errno (pfile
, CPP_DL_ERROR
,
1681 "converting UCN to source character set");
1686 return CPP_HASHNODE (ht_lookup (pfile
->hash_table
,
1687 buf
, bufp
- buf
, HT_ALLOC
));
1690 /* Convert an input buffer (containing the complete contents of one
1691 source file) from INPUT_CHARSET to the source character set. INPUT
1692 points to the input buffer, SIZE is its allocated size, and LEN is
1693 the length of the meaningful data within the buffer. The
1694 translated buffer is returned, *ST_SIZE is set to the length of
1695 the meaningful data within the translated buffer, and *BUFFER_START
1696 is set to the start of the returned buffer. *BUFFER_START may
1697 differ from the return value in the case of a BOM or other ignored
1700 INPUT is expected to have been allocated with xmalloc. This
1701 function will either set *BUFFER_START to INPUT, or free it and set
1702 *BUFFER_START to a pointer to another xmalloc-allocated block of
1705 _cpp_convert_input (cpp_reader
*pfile
, const char *input_charset
,
1706 uchar
*input
, size_t size
, size_t len
,
1707 const unsigned char **buffer_start
, off_t
*st_size
)
1709 struct cset_converter input_cset
;
1710 struct _cpp_strbuf to
;
1711 unsigned char *buffer
;
1713 input_cset
= init_iconv_desc (pfile
, SOURCE_CHARSET
, input_charset
);
1714 if (input_cset
.func
== convert_no_conversion
)
1722 to
.asize
= MAX (65536, len
);
1723 to
.text
= XNEWVEC (uchar
, to
.asize
);
1726 if (!APPLY_CONVERSION (input_cset
, input
, len
, &to
))
1727 cpp_error (pfile
, CPP_DL_ERROR
,
1728 "failure to convert %s to %s",
1729 CPP_OPTION (pfile
, input_charset
), SOURCE_CHARSET
);
1734 /* Clean up the mess. */
1735 if (input_cset
.func
== convert_using_iconv
)
1736 iconv_close (input_cset
.cd
);
1738 /* Resize buffer if we allocated substantially too much, or if we
1739 haven't enough space for the \n-terminator or following
1740 15 bytes of padding (used to quiet warnings from valgrind or
1741 Address Sanitizer, when the optimized lexer accesses aligned
1742 16-byte memory chunks, including the bytes after the malloced,
1743 area, and stops lexing on '\n'). */
1744 if (to
.len
+ 4096 < to
.asize
|| to
.len
+ 16 > to
.asize
)
1745 to
.text
= XRESIZEVEC (uchar
, to
.text
, to
.len
+ 16);
1747 memset (to
.text
+ to
.len
, '\0', 16);
1749 /* If the file is using old-school Mac line endings (\r only),
1750 terminate with another \r, not an \n, so that we do not mistake
1751 the \r\n sequence for a single DOS line ending and erroneously
1752 issue the "No newline at end of file" diagnostic. */
1753 if (to
.len
&& to
.text
[to
.len
- 1] == '\r')
1754 to
.text
[to
.len
] = '\r';
1756 to
.text
[to
.len
] = '\n';
1760 #if HOST_CHARSET == HOST_CHARSET_ASCII
1761 /* The HOST_CHARSET test just above ensures that the source charset
1762 is UTF-8. So, ignore a UTF-8 BOM if we see one. Note that
1763 glib'c UTF-8 iconv() provider (as of glibc 2.7) does not ignore a
1764 BOM -- however, even if it did, we would still need this code due
1765 to the 'convert_no_conversion' case. */
1766 if (to
.len
>= 3 && to
.text
[0] == 0xef && to
.text
[1] == 0xbb
1767 && to
.text
[2] == 0xbf)
1774 *buffer_start
= to
.text
;
1778 /* Decide on the default encoding to assume for input files. */
1780 _cpp_default_encoding (void)
1782 const char *current_encoding
= NULL
;
1784 /* We disable this because the default codeset is 7-bit ASCII on
1785 most platforms, and this causes conversion failures on every
1786 file in GCC that happens to have one of the upper 128 characters
1787 in it -- most likely, as part of the name of a contributor.
1788 We should definitely recognize in-band markers of file encoding,
1790 - the appropriate Unicode byte-order mark (FE FF) to recognize
1791 UTF16 and UCS4 (in both big-endian and little-endian flavors)
1793 - a "#i", "#d", "/ *", "//", " #p" or "#p" (for #pragma) to
1794 distinguish ASCII and EBCDIC.
1795 - now we can parse something like "#pragma GCC encoding <xyz>
1796 on the first line, or even Emacs/VIM's mode line tags (there's
1797 a problem here in that VIM uses the last line, and Emacs has
1798 its more elaborate "local variables" convention).
1799 - investigate whether Java has another common convention, which
1800 would be friendly to support.
1801 (Zack Weinberg and Paolo Bonzini, May 20th 2004) */
1802 #if defined (HAVE_LOCALE_H) && defined (HAVE_LANGINFO_CODESET) && 0
1803 setlocale (LC_CTYPE
, "");
1804 current_encoding
= nl_langinfo (CODESET
);
1806 if (current_encoding
== NULL
|| *current_encoding
== '\0')
1807 current_encoding
= SOURCE_CHARSET
;
1809 return current_encoding
;