Imbuing streams to auto-codecvt [was Re: _M_underflow_common missing]
Brad Spencer
spencer@infointeractive.com
Wed Sep 4 13:19:00 GMT 2002
On Wed, Sep 04, 2002 at 05:07:26PM -0300, Brad Spencer wrote:
> > If you can post complete code samples that demonstrate what you are
> > doing it would be helpful.
>
> I will followup with a complete (small) sample program in a few
> minutes.
This works for my Linux systems because they have wchar_t support in
the library. It should work for any system that has wchar_t support,
AFAICT, but I don't have any more :(
[Attached]
--
------------------------------------------------------------------
Brad Spencer - spencer@infointeractive.com - "It's quite nice..."
Systems Architect | InfoInterActive Corp. | A Canadian AOL Company
-------------- next part --------------
//*****************************************************************************
// Testing code conversion via streams
#include <iostream>
#include <fstream>
#include <locale>
#include <iomanip>
#include <sstream>
#include <string>
#include <vector>
#include <inttypes.h>
using std::cout;
using std::endl;
//=============================================================================
#ifndef _GLIBCPP_USE_WCHAR_T
#error Messed up!!!
#endif // _GLIBCPP_USE_WCHAR_T
//=============================================================================
// My pseudo-library so far...
namespace iiaCodeCvt
{
// Namespace for a bunch of codenames
namespace Codesets
{
// The current names we "support". We don't even really care if the
// values are in the table.
enum Names {
ucs4 = 0,
shift_jis,
ucs4test // TODO: remove this codeset
};
// TODO: debugging output op for those?
};
// Singleton to hold the user-configurable names
class CodesetNames
{
typedef std::vector<std::string> NameTable;
NameTable m_names;
public:
// Set the installation-specific name for the codeset (from configuration
// stored elsewhere, presumably)
void setName(const Codesets::Names codeset,
const std::string &name)
{
const NameTable::size_type index =
static_cast<NameTable::size_type>(codeset);
m_names.resize(index + 1);
m_names[index] = name;
}
// Return the name of a codeset, if configured
// TODO: check exception specification
const char * const getName(const Codesets::Names codeset) const
throw(std::exception)
{
const NameTable::size_type index =
static_cast<NameTable::size_type>(codeset);
return m_names.at(index).c_str();
}
};
// We always have one of these
// TODO: put in implementation file
CodesetNames codesetNames;
// Access the singleton
// TODO: non-inline
inline CodesetNames &getCodesetNames() { return codesetNames; }
// --------------------------------------------------------------------
// Some implementation
// Fix the __enc_traits to a partiular pair of codesets
template<Codesets::Names _InternCS, Codesets::Names _ExternCS>
class codeset_pair_state : public std::__enc_traits
{
typedef std::__enc_traits BaseClass;
public:
// Fix the codesets as part of this type
codeset_pair_state()
: BaseClass(getCodesetNames().getName(_InternCS),
getCodesetNames().getName(_ExternCS))
{
// Finish initialization
BaseClass::_M_init();
}
};
// API: This is the class that the user cares about
template<typename _CharT,
Codesets::Names _InternCS, Codesets::Names _ExternCS>
struct codeset_pair_traits : public std::char_traits<_CharT>
{
// Some basics
typedef codeset_pair_state<_InternCS, _ExternCS> state_type;
};
}
namespace std
{
// API:
// We need to do this, too, or all of the do_* members are not linkable
// because of the way that codecvt is implemented in libstdc++. The user
// should construct this as part of the custom locale being made.
//
// This facet is exactly the same as the real codecvt, but it avoids having
// to duplicate the partial specialization completely and knows an absolute
// minimum about the libstdc++ implementation (essentially, the existance of
// std::__enc_traits and what it is for).
template<typename _InternT, typename _ExternT,
iiaCodeCvt::Codesets::Names _InternCS,
iiaCodeCvt::Codesets::Names _ExternCS>
class codecvt<_InternT, _ExternT,
iiaCodeCvt::codeset_pair_state<_InternCS, _ExternCS> >
: public std::codecvt<_InternT, _ExternT, std::__enc_traits>
{
typedef std::codecvt<_InternT, _ExternT, std::__enc_traits> BaseClass;
public:
// Re-advertise the new state_type so that basic_fstream (and therefore
// basic_filebuf) can see it.
typedef typename iiaCodeCvt::codeset_pair_state<_InternCS, _ExternCS>
state_type;
// Mimic the base class
explicit
codecvt(size_t __refs = 0)
: BaseClass(__refs)
{}
// Mimic the base class
explicit
codecvt(typename BaseClass::__enc_type* __enc, size_t __refs = 0)
: BaseClass(__enc, __refs)
{}
};
}
//----------------------------------------------------------------------
// This would be application code..
typedef iiaCodeCvt::codeset_pair_traits<wchar_t,
iiaCodeCvt::Codesets::ucs4,
iiaCodeCvt::Codesets::shift_jis>
my_char_traits;
//----------------------------------------------------------------------
// Now the library again :(
#if 1
// We have to write special version of underflow... Why?
namespace std
{
// This is an exact copy of the version in gcc-3.2's fstream.cc, except
// for the template specification. Why do I have to do this?
basic_filebuf<my_char_traits::char_type, my_char_traits>::int_type
basic_filebuf<my_char_traits::char_type, my_char_traits>::
_M_underflow_common(bool __bump)
{
int_type __ret = traits_type::eof();
bool __testin = _M_mode & ios_base::in;
bool __testout = _M_mode & ios_base::out;
if (__testin)
{
// Check for pback madness, and if so swich back to the
// normal buffers and jet outta here before expensive
// fileops happen...
if (_M_pback_init)
{
_M_pback_destroy();
if (_M_in_cur < _M_in_end)
return traits_type::to_int_type(*_M_in_cur);
}
// Sync internal and external buffers.
// NB: __testget -> __testput as _M_buf_unified here.
bool __testget = _M_in_cur && _M_in_beg < _M_in_cur;
bool __testinit = _M_is_indeterminate();
if (__testget)
{
if (__testout)
_M_really_overflow();
else if (_M_in_cur != _M_filepos)
_M_file.seekoff(_M_in_cur - _M_filepos,
ios_base::cur, ios_base::in);
}
if (__testinit || __testget)
{
const locale __loc = this->getloc();
const __codecvt_type& __cvt = use_facet<__codecvt_type>(__loc);
streamsize __elen = 0;
streamsize __ilen = 0;
if (__cvt.always_noconv())
{
__elen = _M_file.xsgetn(reinterpret_cast<char*>(_M_in_beg),
_M_buf_size);
__ilen = __elen;
}
else
{
char* __buf = static_cast<char*>(__builtin_alloca(_M_buf_size));
__elen = _M_file.xsgetn(__buf, _M_buf_size);
const char* __eend;
char_type* __iend;
__res_type __r = __cvt.in(_M_state_cur, __buf,
__buf + __elen, __eend, _M_in_beg,
_M_in_beg + _M_buf_size, __iend);
if (__r == codecvt_base::ok)
__ilen = __iend - _M_in_beg;
else
{
// Unwind.
__ilen = 0;
_M_file.seekoff(-__elen, ios_base::cur, ios_base::in);
}
}
if (0 < __ilen)
{
_M_set_determinate(__ilen);
if (__testout)
_M_out_cur = _M_in_cur;
__ret = traits_type::to_int_type(*_M_in_cur);
if (__bump)
_M_in_cur_move(1);
else if (_M_buf_size == 1)
{
// If we are synced with stdio, we have to unget the
// character we just read so that the file pointer
// doesn't move.
_M_file.sys_ungetc(traits_type::to_int_type(*_M_in_cur));
_M_set_indeterminate();
}
}
}
}
_M_last_overflowed = false;
return __ret;
}
}
#endif // 0
//-----------------------------------------------------------------------------
// And the application again
template<typename _String>
void
dumpString(const _String &in)
{
cout << "length=" << in.size() << ": " << endl << std::hex;
for(std::wstring::size_type i = 0; i != in.size(); ++i) {
cout << '[' << i << ']' << '='
<< std::setfill('0') << std::setw(8) << in[i]
<< ' ' << endl;
}
cout << std::dec << std::setfill(' ') << std::setw(0);
}
int
sjis()
{
cout << "Name the codesets" << endl;
// TODO: configuation parameter
// These names work for my particular Linux box...
iiaCodeCvt::getCodesetNames().setName(iiaCodeCvt::Codesets::ucs4,
"UCS-4LE");
iiaCodeCvt::getCodesetNames().setName(iiaCodeCvt::Codesets::shift_jis,
"SHIFT_JIS");
// TODO: remove when codeset is removed
iiaCodeCvt::getCodesetNames().setName(iiaCodeCvt::Codesets::ucs4test,
"UCS-4BE");
// Ok, prepare a locale
cout << "Trying to create locale" << endl;
// Make a locale for that, but use a custom codeset converstion facet
typedef std::codecvt<my_char_traits::char_type, char,
my_char_traits::state_type> my_codecvt;
std::locale loc(std::locale::classic(), new my_codecvt);
// Check the encodings
{
my_codecvt::state_type x;
cout << "internal enc: " << x._M_get_internal_enc() << endl
<< "external enc: " << x._M_get_external_enc() << endl;
}
// Open a file and imbue the locale on the filebuf so it code converts.
// This file contains Shift-JIS encoded characters.
std::basic_ifstream<my_char_traits::char_type, my_char_traits>
file("somefile");
file.rdbuf()->pubimbue(loc);
// file.imbue(loc);
cout << "open" << endl;
// Read a character from the file
while(file.eof() == false) {
std::basic_string<my_char_traits::char_type, my_char_traits> in;
file >> in;
// Dump the hex
cout << "in: ";
dumpString(in);
}
cout << endl;
// Test
std::basic_string<my_char_traits::char_type, my_char_traits>
test(L"Hello world!");
cout << "test: ";
dumpString(test);
return 0;
}
int
main(const int argc, const char * const * const argv)
{
// For reasonable speed
std::ios::sync_with_stdio(false);
try {
return sjis();
} catch(const std::exception &ex) {
cout << "MAIN: Caught \"" << ex.what() << '\"' << endl;
} catch(...) {
cout << "MAIN: Caught unknown exception" << endl;
}
return 0;
}
//*****************************************************************************
More information about the Libstdc++
mailing list