how to implement lexer with more than one state(s) ?

classic Classic list List threaded Threaded
1 message Options
Reply | Threaded
Open this post in threaded view
|

how to implement lexer with more than one state(s) ?

Jens Kallup
Hello,

like the subject says:
how to implement lexer with more than one state?

here the code with the issue:
Jens

#include "includes/mainwindow.h"

#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/qi_skip.hpp>
#include <boost/spirit/include/qi_eol.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/bind.hpp>
#include <boost/ref.hpp>
#include <iostream>
#include <fstream>

int lineno = 1;


using namespace std;

namespace lex   = boost::spirit::lex;
namespace qi    = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;

namespace phx   = boost::phoenix;

using boost::spirit::ascii::space; // use the ASCII space parser
using boost::spirit::ascii::char_;

using boost::phoenix::val;

enum TokenEnum {
     TokPrintLn,
     TokPrintStr
};

struct set_lexer_state
{
     set_lexer_state(char const* state_)
         : state(state_)
     {
         QString str = state_;
         if (str == "PRINTLN") {
             cout << "printer"
                  << endl;
         }
     }

     // This is called by the semantic action handling code during the
lexing
     template <typename Iterator, typename Context>
     void operator()(Iterator const&, Iterator const&
                     , BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&
                     , std::size_t&, Context& ctx) const
     {
         ctx.set_state_name(state.c_str());
     }

     std::string state;
};

struct add_token
{
     // This is called by the semantic action handling code during the
lexing
     template <typename Iterator, typename Context>
     void operator()(Iterator const&, Iterator const&
                     , BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&
                     , std::size_t&, Context& ctx) const
     {
         ctx.set_state_name("PRINTLN");
         set_lexer_state("INITIAL");
     }

     add_token(TokenEnum entype)
     {
         switch (entype)
         {
             case TokPrintStr:
             {
                 std::cout << "11: " << std::endl;
             }   break;
         }
         set_lexer_state("INITIAL");
     }
};

template <typename Lexer>
struct dbase_tokens : lex::lexer<Lexer>
{
     dbase_tokens()
         : dbase_tokens::base_type(lex::match_flags::match_default)
     {
         printLn    = "\\?";

         //cc_comment = "\\/\\/.*";

         defEol     = "[ \\r\\n]*";

         //definition = "^\\n*";
         //stringStr1 = "\\\".*\\\"";
         //stringStr2 = "\\'.*\\'[ \\t\\n]*";
         //stringStr3 = "\\[.*\\][ \\t\\n]*";

         whitespace = "[ \\t\\r\\n]*";

         this->self =
               cc_comment  [ set_lexer_state("COMMENT") ] |
               printLn     [ set_lexer_state("PRINTLN") ]
             ;

         this->self("PRINTLN")
             = defEol      [ set_lexer_state("INITIAL") ]

             //| stringStr1 [ add_token(TokenEnum::TokPrintStr) ]
             //| stringStr2 [ add_token(TokenEnum::TokPrintStr) ]
             //| stringStr3 [ add_token(TokenEnum::TokPrintStr) ]

           //  | cc_comment [ set_lexer_state("COMMENT") ]
           //  | definition [ lex::_pass = lex::pass_flags::pass_ignore ]
             //| whitespace  [ set_lexer_state("INITIAL") ]
             ;

         this->self("COMMENT")
             = defEol     [ set_lexer_state("INITIAL") ]
             //| definition [ lex::_pass = lex::pass_flags::pass_ignore ]
             //| whitespace [ set_lexer_state("INITIAL") ]
             ;
     }

     lex::token_def<lex::omit> cc_comment, defEol,
     printLn,
     whitespace;

     lex::token_def<string> definition,

     stringStr1,
     stringStr2,
     stringStr3;
};

template <typename Iterator>
struct dbase_grammar : public qi::grammar<Iterator>
{
     template <typename TokenDef>

     dbase_grammar(TokenDef const& tok) :
     dbase_grammar::base_type(start)
     {
         start = +symsbols;

         symsbols
               =
               //tok.cc_comment |
               tok.printLn
         ;

     }
     qi::rule<Iterator> start,
     symsbols;
};

bool parseText(QString text)
{
     std::string data(text.toStdString().c_str());
     if (data.size() < 1) {
         QMessageBox::information(0,"Error","No Data for parser.\nABORT.");
         return false;
     }

     typedef std::string::iterator base_iterator_type;
     typedef lex::lexertl::token<
             base_iterator_type, boost::mpl::vector<std::string>
         > token_type;
     typedef lex::lexertl::actor_lexer<token_type> lexer_type;
     typedef dbase_tokens<lexer_type> dbase_tokens;
     typedef dbase_grammar<dbase_tokens::iterator_type> dbase_grammar;

     dbase_tokens  dbase_lexer;
     dbase_grammar dbase_parser(dbase_lexer);

     auto first = data.begin();
     auto last  = data.end();

     bool r = false;

     lex::tokenize(first, last, dbase_lexer);
     first = data.begin();

     r = lex::tokenize_and_parse(first, last, dbase_lexer, dbase_parser);
     //r = qi::phrase_parse(first, last, ini_parser, ascii::space);
     if (r && first == last) {
         cout << "SUCCESS" << endl;
         return true;
     } else {
         cout << "ERROR at " << endl;
         cout << std::string(first, last) << endl;
         cout << "abort." << endl;
     }   return false;
}

------------------------------------------------------------------------------
Attend Shape: An AT&T Tech Expo July 15-16. Meet us at AT&T Park in San
Francisco, CA to explore cutting-edge tech and listen to tech luminaries
present their vision of the future. This family event has something for
everyone, including kids. Get more information and register today.
http://sdm.link/attshape
_______________________________________________
Spirit-general mailing list
[hidden email]
https://lists.sourceforge.net/lists/listinfo/spirit-general