This is the mail archive of the
libstdc++@gcc.gnu.org
mailing list for the libstdc++ project.
[Patch] Codecvt::do_in rewrite
- From: Paolo Carlini <pcarlini at suse dot de>
- To: libstdc++ <libstdc++ at gcc dot gnu dot org>
- Date: Mon, 24 Nov 2003 18:14:12 +0100
- Subject: [Patch] Codecvt::do_in rewrite
Hi,
these are the do_in bits: very same scheme, same performance gain
(~20x) (*):
wchar_t_in.cc C (iconv) 137r 131u 0s 0mem 2pf
wchar_t_in.cc C++ (codecvt) 187r 181u 0s 0mem 2pf
Tested x86-linux, will commit before the end of the (italian) day.
Paolo.
(*) By the way, for complete char-by-char I/O (i.e., wspeed.cc in
PR 11602) we are now doing *much* better than putwc_unlocked, thanks
to the general improvements to filebuf ;)
/////////
2003-11-24 Paolo Carlini <pcarlini@suse.de>
* config/locale/gnu/codecvt_members.cc (codecvt::do_in):
Rewrite, using both mbsnrtowcs and mbrtowc in a loop: the
former is very fast, but stops if encounters a NUL.
(codecvt::do_out): Tweak.
* testsuite/performance/wchar_t_in.cc: New.
diff -prN libstdc++-v3-1/config/locale/gnu/codecvt_members.cc libstdc++-v3/config/locale/gnu/codecvt_members.cc
*** libstdc++-v3-1/config/locale/gnu/codecvt_members.cc Fri Nov 21 01:42:29 2003
--- libstdc++-v3/config/locale/gnu/codecvt_members.cc Mon Nov 24 16:51:54 2003
*************** namespace std
*** 97,105 ****
extern_type __buf[MB_LEN_MAX];
__tmp_state = __state;
const size_t __conv = wcrtomb(__buf, *__from_next, &__tmp_state);
! if (__conv == static_cast<size_t>(-1))
! __ret = error;
! else if (__conv > static_cast<size_t>(__to_end - __to_next))
__ret = partial;
else
{
--- 97,103 ----
extern_type __buf[MB_LEN_MAX];
__tmp_state = __state;
const size_t __conv = wcrtomb(__buf, *__from_next, &__tmp_state);
! if (__conv > static_cast<size_t>(__to_end - __to_next))
__ret = partial;
else
{
*************** namespace std
*** 126,179 ****
intern_type*& __to_next) const
{
result __ret = ok;
- // This temporary state object is neccessary so __state won't be modified
- // if [__from, __from_end) is a partial multibyte character.
state_type __tmp_state(__state);
#if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ > 2)
__c_locale __old = __uselocale(_M_c_locale_codecvt);
#endif
! // Conversion must be done by calling mbrtowc in a loop rather than
! // by calling mbsrtowcs because mbsrtowcs assumes that the input
! // sequence is zero-terminated.
! while (__from < __from_end && __to < __to_end)
{
! size_t __conv = mbrtowc(__to, __from, __from_end - __from,
! &__tmp_state);
if (__conv == static_cast<size_t>(-1))
{
__ret = error;
- break;
}
! else if (__conv == static_cast<size_t>(-2))
{
! // It is unclear what to return in this case (see DR 382).
__ret = partial;
- break;
}
! else if (__conv == 0)
{
! // XXX Probably wrong for stateful encodings
! __conv = 1;
! *__to = L'\0';
}
! __state = __tmp_state;
! __to++;
! __from += __conv;
}
#if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ > 2)
__uselocale(__old);
#endif
- // It is not clear that __from < __from_end implies __ret != ok
- // (see DR 382).
- if (__ret == ok && __from < __from_end)
- __ret = partial;
-
- __from_next = __from;
- __to_next = __to;
return __ret;
}
--- 124,194 ----
intern_type*& __to_next) const
{
result __ret = ok;
state_type __tmp_state(__state);
+
#if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ > 2)
__c_locale __old = __uselocale(_M_c_locale_codecvt);
#endif
! // mbsnrtowcs is *very* fast but stops if encounters NUL characters:
! // in case we store a L'\0' and then continue, in a loop.
! // NB: mbsnrtowcs is a GNU extension
! __from_next = __from;
! __to_next = __to;
! while (__from_next < __from_end && __to_next < __to_end
! && __ret == ok)
{
! const extern_type* __from_chunk_end;
! __from_chunk_end = static_cast<const extern_type*>(memchr(__from_next, '\0',
! __from_end
! - __from_next));
! if (!__from_chunk_end)
! __from_chunk_end = __from_end;
!
! const extern_type* __tmp_from = __from_next;
! const size_t __conv = mbsnrtowcs(__to_next, &__from_next,
! __from_chunk_end - __from_next,
! __to_end - __to_next, &__state);
if (__conv == static_cast<size_t>(-1))
{
+ // In case of error, in order to stop at the exact place we
+ // have to start again from the beginning with a series of
+ // mbrtowc.
+ while (__tmp_from < __from_next)
+ __tmp_from += mbrtowc(__to_next++, __tmp_from,
+ __from_next - __tmp_from, &__tmp_state);
+ __state = __tmp_state;
__ret = error;
}
! else if (__from_next && __from_next < __from_chunk_end)
{
! // It is unclear what to return in this case (see DR 382).
! __to_next += __conv;
__ret = partial;
}
! else
{
! __from_next = __from_chunk_end;
! __to_next += __conv;
}
! if (__from_next < __from_end && __ret == ok)
! {
! if (__to_next < __to_end)
! {
! // XXX Probably wrong for stateful encodings
! ++__from_next;
! *__to_next++ = L'\0';
! }
! else
! __ret = partial;
! }
}
#if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ > 2)
__uselocale(__old);
#endif
return __ret;
}
diff -prN libstdc++-v3-1/testsuite/performance/wchar_t_in.cc libstdc++-v3/testsuite/performance/wchar_t_in.cc
*** libstdc++-v3-1/testsuite/performance/wchar_t_in.cc Thu Jan 1 01:00:00 1970
--- libstdc++-v3/testsuite/performance/wchar_t_in.cc Mon Nov 24 17:50:20 2003
***************
*** 0 ****
--- 1,84 ----
+ // Copyright (C) 2003 Free Software Foundation, Inc.
+ //
+ // This file is part of the GNU ISO C++ Library. This library is free
+ // software; you can redistribute it and/or modify it under the
+ // terms of the GNU General Public License as published by the
+ // Free Software Foundation; either version 2, or (at your option)
+ // any later version.
+
+ // This library is distributed in the hope that it will be useful,
+ // but WITHOUT ANY WARRANTY; without even the implied warranty of
+ // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ // GNU General Public License for more details.
+
+ // You should have received a copy of the GNU General Public License along
+ // with this library; see the file COPYING. If not, write to the Free
+ // Software Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307,
+ // USA.
+
+ // As a special exception, you may use this file as part of a free software
+ // library without restriction. Specifically, if other files instantiate
+ // templates or use macros or inline functions from this file, or you compile
+ // this file and link it with other files to produce an executable, this
+ // file does not by itself cause the resulting executable to be covered by
+ // the GNU General Public License. This exception does not however
+ // invalidate any other reasons why the executable file might be covered by
+ // the GNU General Public License.
+
+ #include <cstdio>
+ #include <cstring>
+ #include <fstream>
+ #include <langinfo.h>
+ #include <iconv.h>
+ #include <testsuite_performance.h>
+
+ // libstdc++/11602 (do_in)
+ int main(int argc, char** argv)
+ {
+ using namespace std;
+ using namespace __gnu_test;
+
+ time_counter time;
+ resource_counter resource;
+ const int iters = 400000;
+
+ wchar_t wbuf[1024];
+ char cbuf[1024];
+
+ memset(cbuf, 'a', 1024);
+
+ // C (iconv)
+ iconv_t cd = iconv_open("WCHAR_T", nl_langinfo(CODESET));
+ start_counters(time, resource);
+ for (int i = 0; i < iters; ++i)
+ {
+ size_t inbytesleft = 1024;
+ size_t outbytesleft = 1024 * sizeof(wchar_t);
+ char* in = cbuf;
+ char* out = reinterpret_cast<char*>(wbuf);
+ iconv(cd, &in, &inbytesleft, &out, &outbytesleft);
+ }
+ stop_counters(time, resource);
+ iconv_close(cd);
+ report_performance(__FILE__, "C (iconv)", time, resource);
+ clear_counters(time, resource);
+
+ // C++ (codecvt)
+ locale loc;
+ const codecvt<wchar_t, char, mbstate_t>& cvt =
+ use_facet<codecvt<wchar_t, char, mbstate_t> >(loc);
+ mbstate_t state;
+ memset(&state, 0, sizeof(state));
+ start_counters(time, resource);
+ for (int i = 0; i < iters; ++i)
+ {
+ const char* from_next;
+ wchar_t* to_next;
+ cvt.in(state, cbuf, cbuf + 1024, from_next,
+ wbuf, wbuf + 1024, to_next);
+ }
+ stop_counters(time, resource);
+ report_performance(__FILE__, "C++ (codecvt)", time, resource);
+
+ return 0;
+ }