How to use Boost Spirit to parse Chinese(unicode utf-16)?

Question

My program does not recognize Chinese. How to use spirit to recognize Chinese? I use wstring and has convert it to utf-16.

Here is my header file:

#pragma once

#define BOOST_SPIRIT_UNICODE 

#include <boost/spirit/include/qi.hpp>  
#include <string>
#include <vector>
#include <map>
using namespace std;



namespace qi = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;


typedef pair<wstring,wstring> WordMeaningType;
typedef vector<WordMeaningType> WordMeaningsType;
typedef pair<wstring,WordMeaningsType> WordType;
typedef vector<WordType> WordListType;

struct WordPaser
    :qi::grammar<wstring::iterator,WordListType(),ascii::space_type > 
{
public:
    qi::rule<wstring::iterator, wstring(),ascii::space_type> mRuleWordPart;
    qi::rule<wstring::iterator, wstring(),ascii::space_type> mRuleWordMeaning;
    qi::rule<wstring::iterator, wstring(),ascii::space_type> mRuleWord;

    qi::rule<wstring::iterator, WordMeaningType(),ascii::space_type> mRulePM;
    qi::rule<wstring::iterator, WordMeaningsType(),ascii::space_type> mRulePMs;
    qi::rule<wstring::iterator, WordType(),ascii::space_type> mRuleCurWPM;

    qi::rule<wstring::iterator, WordListType(),ascii::space_type> mRuleEntrence;


    wstring mCurWord;
    wstring mCurWordPart;
    wstring mCurWordMeaning;
    WordMeaningType mCurPM;
    WordMeaningsType mCurPMs;
    WordType mCurWPM;

    WordPaser();


};

and my CPP file:

#include <boost/tuple/tuple.hpp>  
#include <boost/spirit/include/qi.hpp>  
#include <boost/spirit/include/phoenix.hpp>  
#include "WordPaser.h"

namespace fusion = boost::fusion;  
namespace phoenix = boost::phoenix; 

using qi::_val;  
using qi::_1;  
using qi::_2;  
using qi::lit;  
using qi::lexeme;  
using qi::space;  
using ascii::char_;  
using ascii::string;  
using qi::graph;
using qi::word;

WordPaser::WordPaser()
     : WordPaser::base_type(mRuleEntrence)  
{


    mRuleWord %= lexeme[(+((qi::alpha)[std::wcout<<L'|'<<_1])>>*(qi::alpha|char_('(')|char_(')')))]; 

    mRuleWordPart  %= lexeme[(+(qi::alpha)[std::wcout<<L'@'<<_1]>>*(qi::alpha|char_('/')|char_('.'))[std::wcout<<L'@'<<_1])];  
    mRuleWordMeaning %= lexeme[(+( qi::graph|char_(L'(')|char_(L')')[std::wcout<<L':'<<_1]))];  

    mRulePM = (mRuleWordPart>>mRuleWordMeaning)
        [
            phoenix::bind(&WordMeaningType::first, _val)=_1, 
            phoenix::bind(&WordMeaningType::second, _val)=_2

        ];

    mRulePMs = +(mRulePM
        [
             phoenix::push_back(  _val , _1)
        ]);

    mRuleCurWPM = (mRuleWord>>mRulePMs)
        [
            phoenix::bind(&WordType::first, _val)=_1, 
            phoenix::bind(&WordType::second, _val)=_2            
        ];

    mRuleEntrence = +mRuleCurWPM
        [
            phoenix::push_back(_val, _1)
        ];

}

Answer 1

You should use parsers/skippers from another namespace, not from ascii. I guess, in you case it should be standard_wide .

Answer 2

this code can parse chinese.

  #define BOOST_TEST_DYN_LINK 
  #define BOOST_SPIRIT_USE_PHOENIX_V3 
  #define BOOST_SPIRIT_UNICODE 
  #include <boost/config/warning_disable.hpp>
  #include <boost/spirit/include/qi.hpp>
  #include <boost/spirit/include/support_standard_wide.hpp>
  #include <boost/spirit/include/karma.hpp>
  #include <boost/spirit/include/qi_parse.hpp>
  #include <boost/phoenix.hpp>
  #include <boost/fusion/include/std_pair.hpp>
  #define BOOST_TEST_MODULE MyTest
  #include <boost/test/unit_test.hpp>
  using namespace std;

BOOST_AUTO_TEST_CASE( parse_chinese)
  {
          namespace qi        = boost::spirit::qi  ;
          namespace ascii     = boost::spirit::ascii ;
          namespace encoding  = boost::spirit::unicode;
          namespace px        = boost::phoenix ;
          using namespace qi::labels;
          std::wstring test=L"中国" ;
          std::wstring found ;
          qi::rule<wstring::iterator,wstring(),encoding::space_type> unicode_string;
          unicode_string = * qi::unicode::char_ [_val += _1 ]  ;   
          if( qi::phrase_parse(test.begin(),test.end(),
                      unicode_string,
                      encoding::space,
                      found)
            )   
          {   
              BOOST_CHECK(true);
              //std::cout << "OK:" << utf16to8(found) << std::endl  ;
          }
          else
          {
              BOOST_CHECK(false);
          }
  }