The Battle for Wesnoth  1.19.0-dev
Public Member Functions | Private Types | Private Member Functions | Private Attributes | List of all members
tokenizer Class Reference

Abstract baseclass for the tokenizer. More...

#include <tokenizer.hpp>

Public Member Functions

 tokenizer (std::istream &in)
 
 ~tokenizer ()
 
const tokennext_token ()
 
const tokencurrent_token () const
 
const std::string & textdomain () const
 
const std::string & get_file () const
 
int get_start_line () const
 

Private Types

enum  { TOK_NONE = 0 , TOK_SPACE = 1 , TOK_NUMERIC = 2 , TOK_ALPHA = 4 }
 

Private Member Functions

 tokenizer ()
 
void next_char ()
 
void next_char_fast ()
 
int peek_char ()
 
int char_type (unsigned c) const
 
bool is_space (int c) const
 
bool is_num (int c) const
 
bool is_alnum (int c) const
 
void skip_comment ()
 
bool skip_command (char const *cmd)
 Returns true if the next characters are the one from cmd followed by a space. More...
 

Private Attributes

int current_
 
int lineno_
 
int startlineno_
 
std::string textdomain_
 
std::string file_
 
token token_
 
buffered_istream in_
 
char char_types_ [128]
 

Detailed Description

Abstract baseclass for the tokenizer.

Definition at line 56 of file tokenizer.hpp.

Member Enumeration Documentation

◆ anonymous enum

anonymous enum
private
Enumerator
TOK_NONE 
TOK_SPACE 
TOK_NUMERIC 
TOK_ALPHA 

Definition at line 133 of file tokenizer.hpp.

Constructor & Destructor Documentation

◆ tokenizer() [1/2]

tokenizer::tokenizer ( std::istream &  in)

◆ ~tokenizer()

tokenizer::~tokenizer ( )

Definition at line 45 of file tokenizer.cpp.

References in_, and buffered_istream::stream().

◆ tokenizer() [2/2]

tokenizer::tokenizer ( )
private

Member Function Documentation

◆ char_type()

int tokenizer::char_type ( unsigned  c) const
inlineprivate

Definition at line 141 of file tokenizer.hpp.

References c, and char_types_.

Referenced by is_alnum(), is_num(), and is_space().

◆ current_token()

const token& tokenizer::current_token ( ) const
inline

Definition at line 64 of file tokenizer.hpp.

References token_.

◆ get_file()

const std::string& tokenizer::get_file ( ) const
inline

Definition at line 81 of file tokenizer.hpp.

References file_.

◆ get_start_line()

int tokenizer::get_start_line ( ) const
inline

Definition at line 86 of file tokenizer.hpp.

References startlineno_.

◆ is_alnum()

bool tokenizer::is_alnum ( int  c) const
inlineprivate

Definition at line 156 of file tokenizer.hpp.

References c, char_type(), TOK_ALPHA, TOK_NONE, and TOK_NUMERIC.

Referenced by next_token().

◆ is_num()

bool tokenizer::is_num ( int  c) const
inlineprivate

Definition at line 151 of file tokenizer.hpp.

References c, char_type(), and TOK_NUMERIC.

Referenced by skip_comment().

◆ is_space()

bool tokenizer::is_space ( int  c) const
inlineprivate

Definition at line 146 of file tokenizer.hpp.

References c, char_type(), and TOK_SPACE.

Referenced by next_token(), skip_command(), and skip_comment().

◆ next_char()

void tokenizer::next_char ( )
inlineprivate

Definition at line 97 of file tokenizer.hpp.

References current_, lineno_, and next_char_fast().

Referenced by next_token().

◆ next_char_fast()

void tokenizer::next_char_fast ( )
inlineprivate

Definition at line 104 of file tokenizer.hpp.

References current_, buffered_istream::get(), and in_.

Referenced by next_char(), next_token(), skip_command(), skip_comment(), and tokenizer().

◆ next_token()

const token & tokenizer::next_token ( )

◆ peek_char()

int tokenizer::peek_char ( )
inlineprivate

Definition at line 128 of file tokenizer.hpp.

References in_, and buffered_istream::peek().

Referenced by next_token().

◆ skip_command()

bool tokenizer::skip_command ( char const *  cmd)
private

Returns true if the next characters are the one from cmd followed by a space.

Skips all the matching characters.

Definition at line 162 of file tokenizer.cpp.

References current_, is_space(), and next_char_fast().

Referenced by skip_comment().

◆ skip_comment()

void tokenizer::skip_comment ( )
private

Definition at line 174 of file tokenizer.cpp.

References current_, file_, is_num(), is_space(), lineno_, next_char_fast(), skip_command(), and textdomain_.

Referenced by next_token().

◆ textdomain()

const std::string& tokenizer::textdomain ( ) const
inline

Definition at line 76 of file tokenizer.hpp.

References textdomain_.

Member Data Documentation

◆ char_types_

char tokenizer::char_types_[128]
private

Definition at line 176 of file tokenizer.hpp.

Referenced by char_type(), and tokenizer().

◆ current_

int tokenizer::current_
private

Definition at line 93 of file tokenizer.hpp.

Referenced by next_char(), next_char_fast(), next_token(), skip_command(), and skip_comment().

◆ file_

std::string tokenizer::file_
private

Definition at line 170 of file tokenizer.hpp.

Referenced by get_file(), and skip_comment().

◆ in_

buffered_istream tokenizer::in_
private

Definition at line 175 of file tokenizer.hpp.

Referenced by next_char_fast(), peek_char(), tokenizer(), and ~tokenizer().

◆ lineno_

int tokenizer::lineno_
private

Definition at line 94 of file tokenizer.hpp.

Referenced by next_char(), next_token(), and skip_comment().

◆ startlineno_

int tokenizer::startlineno_
private

Definition at line 95 of file tokenizer.hpp.

Referenced by get_start_line(), and next_token().

◆ textdomain_

std::string tokenizer::textdomain_
private

Definition at line 169 of file tokenizer.hpp.

Referenced by skip_comment(), and textdomain().

◆ token_

token tokenizer::token_
private

Definition at line 171 of file tokenizer.hpp.

Referenced by current_token(), and next_token().


The documentation for this class was generated from the following files: