The Battle for Wesnoth  1.15.1+dev
tokenizer.hpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2004 - 2009 by Philippe Plantier <ayin@anathas.org>
3  Copyright (C) 2010 - 2018 by Guillaume Melquiond <guillaume.melquiond@gmail.com>
4  Part of the Battle for Wesnoth Project https://www.wesnoth.org
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation; either version 2 of the License, or
9  (at your option) any later version.
10  This program is distributed in the hope that it will be useful,
11  but WITHOUT ANY WARRANTY.
12 
13  See the COPYING file for more details.
14 */
15 
16 #pragma once
17 
18 //#define DEBUG_TOKENIZER
19 
20 #include "buffered_istream.hpp"
21 
22 #include <istream>
23 #include <string>
24 
25 struct token
26 {
27  token() :
28  type(END),
29  value()
30  {}
31 
33  {
38 
39  LF = '\n',
40  EQUALS = '=',
41  COMMA = ',',
42  PLUS = '+',
43  SLASH = '/',
44  OPEN_BRACKET = '[',
46  UNDERSCORE = '_',
48  };
49 
51  std::string value;
52 };
53 
54 /** Abstract baseclass for the tokenizer. */
55 class tokenizer
56 {
57 public:
58  tokenizer(std::istream& in);
59  ~tokenizer();
60 
61  const token &next_token();
62 
63  const token &current_token() const
64  {
65  return token_;
66  }
67 
68 #ifdef DEBUG_TOKENIZER
69  const token &previous_token() const
70  {
71  return previous_token_;
72  }
73 #endif
74 
75  const std::string &textdomain() const
76  {
77  return textdomain_;
78  }
79 
80  const std::string &get_file() const
81  {
82  return file_;
83  }
84 
85  int get_start_line() const
86  {
87  return startlineno_;
88  }
89 
90 private:
91  tokenizer();
92  int current_;
93  int lineno_;
95 
96  void next_char()
97  {
98  if (current_ == '\n')
99  ++lineno_;
100  next_char_fast();
101  }
102 
104  {
105  do {
106  current_ = in_.get();
107  } while (current_ == '\r');
108 #if 0
109  /// @todo disabled until the campaign server is fixed
110  if(in_.good()) {
111  current_ = in_.get();
112  if (current_ == '\r')
113  {
114  // we assume that there is only one '\r'
115  if(in_.good()) {
116  current_ = in_.get();
117  } else {
118  current_ = EOF;
119  }
120  }
121  } else {
122  current_ = EOF;
123  }
124 #endif
125  }
126 
127  int peek_char()
128  {
129  return in_.peek();
130  }
131 
132  enum
133  {
134  TOK_NONE = 0,
135  TOK_SPACE = 1,
136  TOK_NUMERIC = 2,
137  TOK_ALPHA = 4
138  };
139 
140  int char_type(unsigned c) const
141  {
142  return c < 128 ? char_types_[c] : 0;
143  }
144 
145  bool is_space(int c) const
146  {
147  return (char_type(c) & TOK_SPACE) == TOK_SPACE;
148  }
149 
150  bool is_num(int c) const
151  {
152  return (char_type(c) & TOK_NUMERIC) == TOK_NUMERIC;
153  }
154 
155  bool is_alnum(int c) const
156  {
157  return (char_type(c) & (TOK_ALPHA | TOK_NUMERIC)) != TOK_NONE;
158  }
159 
160  void skip_comment();
161 
162  /**
163  * Returns true if the next characters are the one from @a cmd
164  * followed by a space. Skips all the matching characters.
165  */
166  bool skip_command(char const *cmd);
167 
168  std::string textdomain_;
169  std::string file_;
171 #ifdef DEBUG_TOKENIZER
172  token previous_token_;
173 #endif
175  char char_types_[128];
176 };
void next_char()
Definition: tokenizer.hpp:96
const std::string & textdomain() const
Definition: tokenizer.hpp:75
token()
Definition: tokenizer.hpp:27
bool is_space(int c) const
Definition: tokenizer.hpp:145
int startlineno_
Definition: tokenizer.hpp:94
Abstract baseclass for the tokenizer.
Definition: tokenizer.hpp:55
Helper class for buffering a std::istream.
void next_char_fast()
Definition: tokenizer.hpp:103
unsigned in
If equal to search_counter, the node is off the list.
int get_start_line() const
Definition: tokenizer.hpp:85
token_type
Definition: tokenizer.hpp:32
std::string file_
Definition: tokenizer.hpp:169
int lineno_
Definition: tokenizer.hpp:93
bool is_alnum(int c) const
Definition: tokenizer.hpp:155
Helper class for buffering a std::istream.
const token & current_token() const
Definition: tokenizer.hpp:63
int peek_char()
Definition: tokenizer.hpp:127
int char_type(unsigned c) const
Definition: tokenizer.hpp:140
std::string textdomain_
Definition: tokenizer.hpp:168
int current_
Definition: tokenizer.hpp:92
bool is_num(int c) const
Definition: tokenizer.hpp:150
buffered_istream in_
Definition: tokenizer.hpp:174
token token_
Definition: tokenizer.hpp:170
mock_char c
const std::string & get_file() const
Definition: tokenizer.hpp:80
std::string value
Definition: tokenizer.hpp:51
token_type type
Definition: tokenizer.hpp:50