How to read an UTF-8 encoded file containing Chinese characters and output them correctly on console?

Question

I am writing a web crawler to fetch some Chinese web files. The fetched files are encoded in utf-8. And I need to read those file to do some parse, such as extracting the UR

Answer 1

This code may help (it was compiled with VC++ 2010). I tested it with an UTF-8 file containing non-latin characters and it seems to work, but I don't know if it will work fine with Chinese characters. Check the following links for more information: _setmode and codecvt_utf8.

#include 
#include 
#include 
#include 
#include 
#include 
#include 

using namespace std;    // Sorry for this!

void read_all_lines(const wchar_t *filename)
{
    wifstream wifs;
    wstring txtline;
    int c = 0;

    wifs.open(filename);
    if(!wifs.is_open())
    {
        wcerr << L"Unable to open file" << endl;
        return;
    }
    // We are going to read an UTF-8 file
    wifs.imbue(locale(wifs.getloc(), new codecvt_utf8()));
    while(getline(wifs, txtline))
        wcout << ++c << L'\t' << txtline << L'\n';
    wcout << endl;
}

int _tmain(int argc, _TCHAR* argv[])
{
    // Console output will be UTF-16 characters
    _setmode(_fileno(stdout), _O_U16TEXT);
    if(argc < 2)
    {
        wcerr << L"Filename expected!" << endl;
        return 1;
    }
    read_all_lines(argv[1]);
    return 0;
}

If Chinese characters don't look as expected, make sure the console is using a font that supports UTF-16 (ie. don't use bitmap fonts).