62 const token &next_token();
69 #ifdef DEBUG_TOKENIZER 70 const token &previous_token()
const 72 return previous_token_;
107 current_ = in_.get();
108 }
while (current_ ==
'\r');
112 current_ = in_.get();
113 if (current_ ==
'\r')
117 current_ = in_.get();
143 return c < 128 ? char_types_[
c] : 0;
148 return (char_type(c) & TOK_SPACE) == TOK_SPACE;
153 return (char_type(c) & TOK_NUMERIC) == TOK_NUMERIC;
158 return (char_type(c) & (TOK_ALPHA | TOK_NUMERIC)) != TOK_NONE;
167 bool skip_command(
char const *cmd);
172 #ifdef DEBUG_TOKENIZER 173 token previous_token_;
176 char char_types_[128];
const std::string & textdomain() const
bool is_space(int c) const
Abstract baseclass for the tokenizer.
Helper class for buffering a std::istream.
unsigned in
If equal to search_counter, the node is off the list.
int get_start_line() const
bool is_alnum(int c) const
Helper class for buffering a std::istream.
const token & current_token() const
int char_type(unsigned c) const
const std::string & get_file() const