The Battle for Wesnoth  1.15.2+dev
preprocessor.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2003 by David White <dave@whitevine.net>
3  Copyright (C) 2005 - 2018 by Guillaume Melquiond <guillaume.melquiond@gmail.com>
4  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation; either version 2 of the License, or
9  (at your option) any later version.
10  This program is distributed in the hope that it will be useful,
11  but WITHOUT ANY WARRANTY.
12 
13  See the COPYING file for more details.
14 */
15 
16 /**
17  * @file
18  * WML preprocessor.
19  */
20 
22 
23 #include "buffered_istream.hpp"
24 #include "config.hpp"
25 #include "filesystem.hpp"
26 #include "log.hpp"
28 #include "serialization/parser.hpp"
30 #include "game_version.hpp"
31 #include "wesconfig.h"
32 #include "deprecation.hpp"
33 
34 #include <stdexcept>
35 #include <deque>
36 
37 static lg::log_domain log_preprocessor("preprocessor");
38 #define ERR_PREPROC LOG_STREAM(err, log_preprocessor)
39 #define WRN_PREPROC LOG_STREAM(warn, log_preprocessor)
40 #define LOG_PREPROC LOG_STREAM(info, log_preprocessor)
41 #define DBG_PREPROC LOG_STREAM(debug, log_preprocessor)
42 
43 static const std::string current_file_str = "CURRENT_FILE";
44 static const std::string current_dir_str = "CURRENT_DIRECTORY";
45 static const std::string left_curly_str = "LEFT_BRACE";
46 static const std::string right_curly_str = "RIGHT_BRACE";
47 
48 // map associating each filename encountered to a number
49 static std::map<std::string, int> file_number_map;
50 
51 static bool encode_filename = true;
52 
53 static std::string preprocessor_error_detail_prefix = "\n ";
54 
55 static const char OUTPUT_SEPARATOR = '\xFE';
56 
57 // get filename associated to this code
58 static std::string get_filename(const std::string& file_code)
59 {
60  if(!encode_filename) {
61  return file_code;
62  }
63 
64  std::stringstream s;
65  s << file_code;
66  int n = 0;
67  s >> std::hex >> n;
68 
69  for(const auto& p : file_number_map) {
70  if(p.second == n) {
71  return p.first;
72  }
73  }
74 
75  return "<unknown>";
76 }
77 
78 // Get code associated to this filename
79 static std::string get_file_code(const std::string& filename)
80 {
81  if(!encode_filename) {
82  return filename;
83  }
84 
85  // Current number of encountered filenames
86  static int current_file_number = 0;
87 
88  int& fnum = file_number_map[utils::escape(filename, " \\")];
89  if(fnum == 0) {
90  fnum = ++current_file_number;
91  }
92 
93  std::ostringstream shex;
94  shex << std::hex << fnum;
95 
96  return shex.str();
97 }
98 
99 // decode the filenames placed in a location
100 static std::string get_location(const std::string& loc)
101 {
102  std::string res;
103  std::vector<std::string> pos = utils::quoted_split(loc, ' ');
104 
105  if(pos.empty()) {
106  return res;
107  }
108 
109  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
110  while(true) {
111  res += get_filename(*(i++));
112 
113  if(i == end) {
114  break;
115  }
116 
117  res += ' ';
118  res += *(i++);
119 
120  if(i == end) {
121  break;
122  }
123 
124  res += ' ';
125  }
126 
127  return res;
128 }
129 
130 
131 // ==================================================================================
132 // PREPROC_DEFINE IMPLEMENTATION
133 // ==================================================================================
134 
136 {
137  return value == v.value && arguments == v.arguments;
138 }
139 
141 {
142  if(location < v.location) {
143  return true;
144  }
145 
146  if(v.location < location) {
147  return false;
148  }
149 
150  return linenum < v.linenum;
151 }
152 
153 void preproc_define::write_argument(config_writer& writer, const std::string& arg) const
154 {
155  const std::string key = "argument";
156 
157  writer.open_child(key);
158 
159  writer.write_key_val("name", arg);
160  writer.close_child(key);
161 }
162 
163 void preproc_define::write(config_writer& writer, const std::string& name) const
164 {
165  const std::string key = "preproc_define";
166  writer.open_child(key);
167 
168  writer.write_key_val("name", name);
169  writer.write_key_val("value", value);
170  writer.write_key_val("textdomain", textdomain);
171  writer.write_key_val("linenum", std::to_string(linenum));
172  writer.write_key_val("location", get_location(location));
173 
174  for(const std::string& arg : arguments) {
175  write_argument(writer, arg);
176  }
177 
178  writer.close_child(key);
179 }
180 
182 {
183  arguments.push_back(cfg["name"]);
184 }
185 
186 void preproc_define::read(const config& cfg)
187 {
188  value = cfg["value"].str();
189  textdomain = cfg["textdomain"].str();
190  linenum = cfg["linenum"];
191  location = cfg["location"].str();
192 
193  for(const config& arg : cfg.child_range("argument")) {
194  read_argument(arg);
195  }
196 }
197 
198 preproc_map::value_type preproc_define::read_pair(const config& cfg)
199 {
200  preproc_define second;
201  second.read(cfg);
202 
203  return preproc_map::value_type(cfg["name"], second);
204 }
205 
206 std::ostream& operator<<(std::ostream& stream, const preproc_define& def)
207 {
208  return stream << "value: " << def.value << " arguments: " << def.location;
209 }
210 
211 std::ostream& operator<<(std::ostream& stream, const preproc_map::value_type& def)
212 {
213  return stream << def.second;
214 }
215 
216 // ==================================================================================
217 // PREPROCESSOR BASE
218 // ==================================================================================
219 
221 
222 /**
223  * Base class for preprocessing an input.
224  */
226 {
228 
229 protected:
230  /**
231  * Sets up a new preprocessor for stream buffer \a t.
232  * Saves the current preprocessing context of #parent_. It will be automatically restored on destruction.
233  *
234  * It relies on preprocessor_streambuf so it's implemented after that class is declared.
235  */
237 
239 
240 public:
241  virtual ~preprocessor()
242  {
243  }
244 
245  /** Allows specifying any actions that need to be called after the constructor completes. */
246  virtual void init()
247  {
248  }
249 
250  /**
251  * Preprocesses and sends some text to the #parent_ buffer.
252  * @return false when the input has no data left.
253  */
254  virtual bool get_chunk() = 0;
255 
256  enum MODE { NO_PARSING, PARSES_FILE, PARSES_MACRO };
257 
258  /** Returns the appropriate parsing mode for this preprocessor. */
259  virtual MODE parse_mode()
260  {
261  return NO_PARSING;
262  }
263 
264 private:
265  std::string old_textdomain_;
266  std::string old_location_;
267 
269 };
270 
271 
272 // ==================================================================================
273 // PREPROCESSOR BUFFER
274 // ==================================================================================
275 
276 /**
277  * Target for sending preprocessed output.
278  * Objects of this class can be plugged into an STL stream.
279  */
280 class preprocessor_streambuf : public std::streambuf
281 {
282 public:
284  : std::streambuf()
285  , out_buffer_("")
286  , buffer_()
287  , preprocessor_queue_()
288  , defines_(def)
289  , default_defines_()
290  , textdomain_(PACKAGE)
291  , location_("")
292  , linenum_(0)
293  , quoted_(false)
294  {
295  }
296 
297  /** Decodes the filenames placed in a location. */
298  std::string get_current_file();
299 
300  void error(const std::string&, int);
301  void warning(const std::string&, int);
302 
303  template<typename T, typename... A>
304  void add_preprocessor(A&&... args)
305  {
306  preprocessor_queue_.emplace_back(new T(*this, std::forward<A>(args)...));
307  preprocessor_queue_.back()->init();
308  }
309 
311  {
312  preprocessor_queue_.pop_back();
313  }
314 
315  int depth() const
316  {
317  return preprocessor_queue_.size();
318  }
319 
321  {
322  return preprocessor_queue_.empty() ? nullptr : preprocessor_queue_.back().get();
323  }
324 
325 private:
327  : std::streambuf()
328  , out_buffer_("")
329  , buffer_()
330  , preprocessor_queue_()
331  , defines_(t.defines_)
332  , default_defines_()
333  , textdomain_(PACKAGE)
334  , location_("")
335  , linenum_(0)
336  , quoted_(t.quoted_)
337  {
338  }
339 
340  /** Inherited from basic_streambuf. */
341  virtual int underflow() override;
342 
343  void restore_old_preprocessor();
344 
345  /** Buffer read by the STL stream. */
346  std::string out_buffer_;
347 
348  /** Buffer filled by the #current_ preprocessor. */
349  std::stringstream buffer_;
350 
351  /** Input preprocessor queue. */
352  std::deque<std::unique_ptr<preprocessor>> preprocessor_queue_;
353 
356 
357  std::string textdomain_;
358  std::string location_;
359 
360  int linenum_;
361 
362  /**
363  * Set to true if one preprocessor for this target started to read a string.
364  * Deeper-nested preprocessors are then forbidden to.
365  */
366  bool quoted_;
367 
368  friend class preprocessor;
369  friend class preprocessor_file;
370  friend class preprocessor_data;
372 };
373 
374 /** Preprocessor constructor. */
376  : parent_(t)
377  , old_textdomain_(t.textdomain_)
378  , old_location_(t.location_)
379  , old_linenum_(t.linenum_)
380 {
381 }
382 
383 /**
384  * Called by an STL stream whenever it has reached the end of #out_buffer_.
385  * Fills #buffer_ by calling the #current_ preprocessor, then copies its
386  * content into #out_buffer_.
387  * @return the first character of #out_buffer_ if any, EOF otherwise.
388  */
390 {
391  unsigned sz = 0;
392  if(char* gp = gptr()) {
393  if(gp < egptr()) {
394  // Sanity check: the internal buffer has not been totally consumed,
395  // should we force the caller to use what remains first?
396  return *gp;
397  }
398 
399  // The buffer has been completely read; fill it again.
400  // Keep part of the previous buffer, to ensure putback capabilities.
401  sz = out_buffer_.size();
402  buffer_.str(std::string());
403 
404  if(sz > 3) {
405  buffer_ << out_buffer_.substr(sz - 3);
406  sz = 3;
407  } else {
408  buffer_ << out_buffer_;
409  }
410  } else {
411  // The internal get-data pointer is null
412  }
413 
414  const int desired_fill_amount = 2000;
415 
416  while(current() && buffer_.rdbuf()->in_avail() < desired_fill_amount) {
417  // Process files and data chunks until the desired buffer size is reached
418  if(!current()->get_chunk()) {
419  // Drop the current preprocessor item from the queue.
420  restore_old_preprocessor();
421  }
422  }
423 
424  // Update the internal state and data pointers
425  out_buffer_ = buffer_.str();
426  if(out_buffer_.empty()) {
427  return EOF;
428  }
429 
430  char* begin = &*out_buffer_.begin();
431  unsigned bs = out_buffer_.size();
432 
433  setg(begin, begin + sz, begin + bs);
434 
435  if(sz >= bs) {
436  return EOF;
437  }
438 
439  return static_cast<unsigned char>(*(begin + sz));
440 }
441 
442 /**
443 * Restores the old preprocessing context.
444 * Appends location and domain directives to the buffer, so that the parser
445 * notices these changes.
446 */
448 {
449  preprocessor* current = this->current();
450 
451  if(!current->old_location_.empty()) {
452  buffer_ << OUTPUT_SEPARATOR << "line " << current->old_linenum_ << ' ' << current->old_location_ << '\n';
453  }
454 
455  if(!current->old_textdomain_.empty() && textdomain_ != current->old_textdomain_) {
456  buffer_ << OUTPUT_SEPARATOR << "textdomain " << current->old_textdomain_ << '\n';
457  }
458 
459  location_ = current->old_location_;
460  linenum_ = current->old_linenum_;
461  textdomain_ = current->old_textdomain_;
462 
463  // Drop the preprocessor from the queue.
464  drop_preprocessor();
465 }
466 
468 {
469  unsigned nested_level = 0;
470 
471  preprocessor* pre = nullptr;
472 
473  // Iterate backwards over queue to get the last non-macro preprocessor.
474  for(auto p = preprocessor_queue_.rbegin(); p != preprocessor_queue_.rend(); ++p) {
475  pre = p->get();
476 
477  if(!pre || pre->parse_mode() == preprocessor::PARSES_FILE) {
478  break;
479  }
480 
481  if(pre->parse_mode() == preprocessor::PARSES_MACRO) {
482  ++nested_level;
483  }
484  }
485 
486  std::string res;
487  std::vector<std::string> pos = utils::quoted_split(location_, ' ');
488 
489  if(pos.size() <= 2 * nested_level) {
490  return res;
491  }
492 
493  return get_filename(pos[2 * nested_level]);
494 }
495 
496 std::string lineno_string(const std::string& lineno)
497 {
498  std::vector<std::string> pos = utils::quoted_split(lineno, ' ');
499  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
500  std::string included_from = preprocessor_error_detail_prefix + "included from ";
501  std::string res;
502 
503  while(i != end) {
504  const std::string& line = *(i++);
505 
506  if(!res.empty()) {
507  res += included_from;
508  }
509 
510  if(i != end) {
511  res += get_filename(*(i++));
512  } else {
513  res += "<unknown>";
514  }
515 
516  res += ':' + line;
517  }
518 
519  if(res.empty()) {
520  res = "???";
521  }
522 
523  return res;
524 }
525 
526 void preprocessor_streambuf::error(const std::string& error_type, int l)
527 {
528  std::string position, error;
529  std::ostringstream pos;
530 
531  pos << l << ' ' << location_;
532  position = lineno_string(pos.str());
533 
534  error = error_type + '\n';
535  error += "at " + position;
536 
537  ERR_PREPROC << error << '\n';
538 
539  throw preproc_config::error(error);
540 }
541 
542 void preprocessor_streambuf::warning(const std::string& warning_type, int l)
543 {
544  std::string position, warning;
545  std::ostringstream pos;
546 
547  pos << l << ' ' << location_;
548  position = lineno_string(pos.str());
549 
550  warning = warning_type + '\n';
551  warning += "at " + position;
552 
553  WRN_PREPROC << warning << '\n';
554 }
555 
556 
557 // ==================================================================================
558 // PREPROCESSOR FILE
559 // ==================================================================================
560 
561 /**
562  * Specialized preprocessor for handling a file or a set of files.
563  * A preprocessor_file object is created when a preprocessor encounters an
564  * inclusion directive that resolves to a file or directory, e.g. '{themes/}'.
565  */
567 {
568 public:
569  /** Constructor. It relies on preprocessor_data so it's implemented after that class is declared. */
570  preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index = -1);
571 
572  virtual void init() override;
573 
574  /**
575  * Inserts and processes the next file in the list of included files.
576  * @return false if there is no next file.
577  */
578  virtual bool get_chunk() override
579  {
580  while(pos_ != end_) {
581  const std::string& name = *(pos_++);
582  unsigned sz = name.size();
583 
584  // Use reverse iterator to optimize testing
585  if(sz < 5 || !std::equal(name.rbegin(), name.rbegin() + 4, "gfc.")) {
586  continue;
587  }
588 
590  return true;
591  }
592 
593  return false;
594  }
595 
596 private:
597  std::vector<std::string> files_;
598  std::vector<std::string>::const_iterator pos_, end_;
599 
600  const std::string& name_;
601 
603 };
604 
605 
606 // ==================================================================================
607 // PREPROCESSOR DATA
608 // ==================================================================================
609 
610 /**
611  * Specialized preprocessor for handling any kind of input stream.
612  * This is the core of the preprocessor.
613  */
615 {
616  /** Description of a preprocessing chunk. */
617  struct token_desc
618  {
619  enum TOKEN_TYPE {
620  START, // Toplevel
621  PROCESS_IF, // Processing the "if" branch of a ifdef/ifndef (the "else" branch will be skipped)
622  PROCESS_ELSE, // Processing the "else" branch of a ifdef/ifndef
623  SKIP_IF, // Skipping the "if" branch of a ifdef/ifndef (the "else" branch, if any, will be processed)
624  SKIP_ELSE, // Skipping the "else" branch of a ifdef/ifndef
625  STRING, // Processing a string
626  VERBATIM, // Processing a verbatim string
627  MACRO_SPACE, // Processing between chunks of a macro call (skip spaces)
628  MACRO_CHUNK, // Processing inside a chunk of a macro call (stop on space or '(')
629  MACRO_PARENS // Processing a parenthesized macro argument
630  };
631 
632  token_desc(TOKEN_TYPE type, const int stack_pos, const int linenum)
633  : type(type)
634  , stack_pos(stack_pos)
635  , linenum(linenum)
636  {
637  }
638 
640 
641  /** Starting position in #strings_ of the delayed text for this chunk. */
643  int linenum;
644  };
645 
646  /**
647  * Manages the lifetime of the @c std::istream pointer we own.
648  *
649  * Since @ref in_ uses the stream as well this object must be created
650  * before @ref in_ and destroyed after @ref in_ is destroyed.
651  */
653 
654  /** Input stream. */
656 
657  std::string directory_;
658 
659  /** Buffer for delayed input processing. */
660  std::vector<std::string> strings_;
661 
662  /** Mapping of macro arguments to their content. */
663  std::unique_ptr<std::map<std::string, std::string>> local_defines_;
664 
665  /** Stack of nested preprocessing chunks. */
666  std::vector<token_desc> tokens_;
667 
668  /**
669  * Set to true whenever input tokens cannot be directly sent to the target
670  * buffer. For instance, this happens with macro arguments. In that case,
671  * the output is redirected toward #strings_ until it can be processed.
672  */
674 
675  /**
676  * Non-zero when the preprocessor has to skip some input text.
677  * Increased whenever entering a conditional branch that is not useful,
678  * e.g. a ifdef that evaluates to false.
679  */
681  int linenum_;
682 
683  /** True iff we are currently parsing a macros content, otherwise false. */
685 
686  std::string read_word();
687  std::string read_line();
688  std::string read_rest_of_line();
689 
690  void skip_spaces();
691  void skip_eol();
692  void push_token(token_desc::TOKEN_TYPE);
693  void pop_token();
694  void put(char);
695  void put(const std::string& /*, int change_line = 0 */);
696  void conditional_skip(bool skip);
697 
698 public:
701  const std::string& history,
702  const std::string& name,
703  int line,
704  const std::string& dir,
705  const std::string& domain,
706  std::unique_ptr<std::map<std::string, std::string>> defines,
707  bool is_define = false);
708 
709  virtual bool get_chunk() override;
710 
711  virtual preprocessor::MODE parse_mode() override
712  {
713  return is_define_ ? PARSES_MACRO : PARSES_FILE;
714  }
715 
720 };
721 
723 {
724  throw std::logic_error("don't compare tokens with characters");
725 }
726 
728 {
729  return rhs == lhs;
730 }
731 
733 {
734  return !(lhs == rhs);
735 }
736 
738 {
739  return rhs != lhs;
740 }
741 
742 /** preprocessor_file constructor. */
743 preprocessor_file::preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index)
744  : preprocessor(t)
745  , files_()
746  , pos_()
747  , end_()
748  , name_(name)
749  , is_directory_(filesystem::is_directory(name))
750 {
751  if(is_directory_) {
752  filesystem::get_files_in_dir(name, &files_, nullptr,
756  );
757 
758  for(const std::string& fname : files_) {
759  std::size_t cpos = fname.rfind(" ");
760 
761  if(cpos != std::string::npos && cpos >= symbol_index) {
762  std::stringstream ss;
763  ss << "Found filename containing whitespace: '" << filesystem::base_name(fname)
764  << "' in included directory '" << name << "'.\nThe included symbol probably looks similar to '"
765  << filesystem::directory_name(fname.substr(symbol_index)) << "'";
766 
767  // TODO: find a real linenumber
768  parent_.error(ss.str(), -1);
769  }
770  }
771  } else {
772  // Handled in the init() function.
773  }
774 
775  pos_ = files_.begin();
776  end_ = files_.end();
777 }
778 
780 {
781  if(is_directory_) {
782  return;
783  }
784 
786 
787  if(!file_stream->good()) {
788  ERR_PREPROC << "Could not open file " << name_ << std::endl;
789  } else {
790  parent_.add_preprocessor<preprocessor_data>(std::move(file_stream), "",
793  );
794  }
795 }
796 
799  const std::string& history,
800  const std::string& name,
801  int linenum,
802  const std::string& directory,
803  const std::string& domain,
804  std::unique_ptr<std::map<std::string, std::string>> defines,
805  bool is_define)
806  : preprocessor(t)
807  , in_scope_(std::move(i))
808  , in_(*in_scope_)
809  , directory_(directory)
810  , strings_()
811  , local_defines_(std::move(defines))
812  , tokens_()
813  , slowpath_(0)
814  , skipping_(0)
815  , linenum_(linenum)
816  , is_define_(is_define)
817 {
818  std::ostringstream s;
819  s << history;
820 
821  if(!name.empty()) {
822  if(!history.empty()) {
823  s << ' ';
824  }
825 
826  s << get_file_code(name);
827  }
828 
829  if(!t.location_.empty()) {
830  s << ' ' << t.linenum_ << ' ' << t.location_;
831  }
832 
833  t.location_ = s.str();
834  t.linenum_ = linenum;
835 
836  t.buffer_ << OUTPUT_SEPARATOR << "line " << linenum << ' ' << t.location_ << '\n';
837 
838  if(t.textdomain_ != domain) {
839  t.buffer_ << OUTPUT_SEPARATOR << "textdomain " << domain << '\n';
840  t.textdomain_ = domain;
841  }
842 
844 }
845 
847 {
848  tokens_.emplace_back(t, strings_.size(), linenum_);
849 
850  if(t == token_desc::MACRO_SPACE) {
851  // Macro expansions do not have any associated storage at start.
852  return;
853  } else if(t == token_desc::STRING || t == token_desc::VERBATIM) {
854  /* Quoted strings are always inlined in the parent token. So
855  * they need neither storage nor metadata, unless the parent
856  * token is a macro expansion.
857  */
858  token_desc::TOKEN_TYPE& outer_type = tokens_[tokens_.size() - 2].type;
859  if(outer_type != token_desc::MACRO_SPACE) {
860  return;
861  }
862 
863  outer_type = token_desc::MACRO_CHUNK;
864  tokens_.back().stack_pos = strings_.size() + 1;
865  }
866 
867  std::ostringstream s;
868  if(!skipping_ && slowpath_) {
869  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
870  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
871  }
872 
873  strings_.push_back(s.str());
874 }
875 
877 {
878  token_desc::TOKEN_TYPE inner_type = tokens_.back().type;
879  unsigned stack_pos = tokens_.back().stack_pos;
880 
881  tokens_.pop_back();
882 
883  token_desc::TOKEN_TYPE& outer_type = tokens_.back().type;
884 
885  if(inner_type == token_desc::MACRO_PARENS) {
886  // Parenthesized macro arguments are left on the stack.
887  assert(outer_type == token_desc::MACRO_SPACE);
888  return;
889  }
890 
891  if(inner_type == token_desc::STRING || inner_type == token_desc::VERBATIM) {
892  // Quoted strings are always inlined.
893  assert(stack_pos == strings_.size());
894  return;
895  }
896 
897  if(outer_type == token_desc::MACRO_SPACE) {
898  /* A macro expansion does not have any associated storage.
899  * Instead, storage of the inner token is not discarded
900  * but kept as a new macro argument. But if the inner token
901  * was a macro expansion, it is about to be appended, so
902  * prepare for it.
903  */
904  if(inner_type == token_desc::MACRO_SPACE || inner_type == token_desc::MACRO_CHUNK) {
905  strings_.erase(strings_.begin() + stack_pos, strings_.end());
906  strings_.emplace_back();
907  }
908 
909  assert(stack_pos + 1 == strings_.size());
910  outer_type = token_desc::MACRO_CHUNK;
911 
912  return;
913  }
914 
915  strings_.erase(strings_.begin() + stack_pos, strings_.end());
916 }
917 
919 {
920  for(;;) {
921  int c = in_.peek();
922 
923  if(in_.eof() || (c != ' ' && c != '\t')) {
924  return;
925  }
926 
927  in_.get();
928  }
929 }
930 
932 {
933  for(;;) {
934  int c = in_.get();
935 
936  if(c == '\n') {
937  ++linenum_;
938  return;
939  }
940 
941  if(in_.eof()) {
942  return;
943  }
944  }
945 }
946 
948 {
949  std::string res;
950 
951  for(;;) {
952  int c = in_.peek();
953 
954  if(c == preprocessor_streambuf::traits_type::eof() || utils::portable_isspace(c)) {
955  // DBG_PREPROC << "(" << res << ")\n";
956  return res;
957  }
958 
959  in_.get();
960  res += static_cast<char>(c);
961  }
962 }
963 
965 {
966  std::string res;
967 
968  for(;;) {
969  int c = in_.get();
970 
971  if(c == '\n') {
972  ++linenum_;
973  return res;
974  }
975 
976  if(in_.eof()) {
977  return res;
978  }
979 
980  if(c != '\r') {
981  res += static_cast<char>(c);
982  }
983  }
984 }
985 
987 {
988  std::string res;
989 
990  while(in_.peek() != '\n' && !in_.eof()) {
991  int c = in_.get();
992 
993  if(c != '\r') {
994  res += static_cast<char>(c);
995  }
996  }
997 
998  return res;
999 }
1000 
1002 {
1003  if(skipping_) {
1004  return;
1005  }
1006 
1007  if(slowpath_) {
1008  strings_.back() += c;
1009  return;
1010  }
1011 
1012  int cond_linenum = c == '\n' ? linenum_ - 1 : linenum_;
1013 
1014  if(unsigned diff = cond_linenum - parent_.linenum_) {
1015  parent_.linenum_ = cond_linenum;
1016 
1017  if(diff <= parent_.location_.size() + 11) {
1018  parent_.buffer_ << std::string(diff, '\n');
1019  } else {
1020  parent_.buffer_ << OUTPUT_SEPARATOR << "line " << parent_.linenum_ << ' ' << parent_.location_ << '\n';
1021  }
1022  }
1023 
1024  if(c == '\n') {
1025  ++parent_.linenum_;
1026  }
1027 
1028  parent_.buffer_ << c;
1029 }
1030 
1031 void preprocessor_data::put(const std::string& s /*, int line_change*/)
1032 {
1033  if(skipping_) {
1034  return;
1035  }
1036 
1037  if(slowpath_) {
1038  strings_.back() += s;
1039  return;
1040  }
1041 
1042  parent_.buffer_ << s;
1043  // parent_.linenum_ += line_change;
1044 }
1045 
1047 {
1048  if(skip) {
1049  ++skipping_;
1050  }
1051 
1053 }
1054 
1056 {
1057  char c = static_cast<char>(in_.get());
1058  token_desc& token = tokens_.back();
1059 
1060  if(in_.eof()) {
1061  // The end of file was reached.
1062  // Make sure we don't have any incomplete tokens.
1063  char const* s;
1064 
1065  switch(token.type) {
1066  case token_desc::START:
1067  return false; // everything is fine
1069  case token_desc::SKIP_IF:
1071  case token_desc::SKIP_ELSE:
1072  s = "#ifdef or #ifndef";
1073  break;
1074  case token_desc::STRING:
1075  s = "Quoted string";
1076  break;
1077  case token_desc::VERBATIM:
1078  s = "Verbatim string";
1079  break;
1082  s = "Macro substitution";
1083  break;
1085  s = "Macro argument";
1086  break;
1087  default:
1088  s = "???";
1089  }
1090 
1091  parent_.error(std::string(s) + " not terminated", token.linenum);
1092  }
1093 
1094  if(c == '\n') {
1095  ++linenum_;
1096  }
1097 
1098  if(c == OUTPUT_SEPARATOR) {
1099  std::string buffer(1, c);
1100 
1101  for(;;) {
1102  char d = static_cast<char>(in_.get());
1103 
1104  if(in_.eof() || d == '\n') {
1105  break;
1106  }
1107 
1108  buffer += d;
1109  }
1110 
1111  buffer += '\n';
1112  // line_change = 1-1 = 0
1113  put(buffer);
1114  } else if(token.type == token_desc::VERBATIM) {
1115  put(c);
1116 
1117  if(c == '>' && in_.peek() == '>') {
1118  put(in_.get());
1119  pop_token();
1120  }
1121  } else if(c == '<' && in_.peek() == '<') {
1122  in_.get();
1124  put('<');
1125  put('<');
1126  } else if(c == '"') {
1127  if(token.type == token_desc::STRING) {
1128  parent_.quoted_ = false;
1129  put(c);
1130  pop_token();
1131  } else if(!parent_.quoted_) {
1132  parent_.quoted_ = true;
1134  put(c);
1135  } else {
1136  parent_.error("Nested quoted string", linenum_);
1137  }
1138  } else if(c == '{') {
1140  ++slowpath_;
1141  } else if(c == ')' && token.type == token_desc::MACRO_PARENS) {
1142  pop_token();
1143  } else if(c == '#' && !parent_.quoted_) {
1144  std::string command = read_word();
1145  bool comment = false;
1146 
1147  if(command == "define") {
1148  skip_spaces();
1149  int linenum = linenum_;
1150  std::vector<std::string> items = utils::split(read_line(), ' ');
1151  std::map<std::string, std::string> optargs;
1152 
1153  if(items.empty()) {
1154  parent_.error("No macro name found after #define directive", linenum);
1155  }
1156 
1157  std::string symbol = items.front();
1158  items.erase(items.begin());
1159  int found_arg = 0, found_enddef = 0, found_deprecate = 0;
1160  boost::optional<DEP_LEVEL> deprecation_level = boost::none;
1161  std::string buffer, deprecation_detail;
1162  version_info deprecation_version = game_config::wesnoth_version;
1163  for(;;) {
1164  if(in_.eof())
1165  break;
1166  char d = static_cast<char>(in_.get());
1167  if(d == '\n')
1168  ++linenum_;
1169  buffer += d;
1170  if(d == '#') {
1171  if(in_.peek() == 'a') {
1172  found_arg = 1;
1173  } else if(in_.peek() == 'd') {
1174  found_deprecate = 1;
1175  } else {
1176  found_enddef = 1;
1177  }
1178  } else {
1179  if(found_arg > 0 && ++found_arg == 4) {
1180  if(std::equal(buffer.end() - 3, buffer.end(), "arg")) {
1181  buffer.erase(buffer.end() - 4, buffer.end());
1182 
1183  skip_spaces();
1184  std::string argname = read_word();
1185  skip_eol();
1186 
1187  std::string argbuffer;
1188 
1189  int found_endarg = 0;
1190  for(;;) {
1191  if(in_.eof()) {
1192  break;
1193  }
1194 
1195  char e = static_cast<char>(in_.get());
1196  if(e == '\n') {
1197  ++linenum_;
1198  }
1199 
1200  argbuffer += e;
1201 
1202  if(e == '#') {
1203  found_endarg = 1;
1204  } else if(found_endarg > 0 && ++found_endarg == 7) {
1205  if(std::equal(argbuffer.end() - 6, argbuffer.end(), "endarg")) {
1206  argbuffer.erase(argbuffer.end() - 7, argbuffer.end());
1207  optargs[argname] = argbuffer;
1208  skip_eol();
1209  break;
1210  } else {
1211  parent_.error("Unterminated #arg definition", linenum_);
1212  }
1213  }
1214  }
1215  }
1216  }
1217 
1218  if(found_deprecate > 0 && ++found_deprecate == 11) {
1219  if(std::equal(buffer.end() - 10, buffer.end(), "deprecated")) {
1220  buffer.erase(buffer.end() - 11, buffer.end());
1221  skip_spaces();
1222  try {
1223  DEP_LEVEL level = DEP_LEVEL(std::stoi(read_word()));
1224  if(deprecation_level) {
1225  deprecation_level = std::max(*deprecation_level, level);
1226  } else {
1227  deprecation_level = level;
1228  }
1229  } catch(const std::invalid_argument&) {
1230  // Meh, fall back to default of PREEMPTIVE...
1231  deprecation_level = DEP_LEVEL::PREEMPTIVE;
1232  }
1233  deprecation_version = game_config::wesnoth_version;
1234  if(deprecation_level == DEP_LEVEL::PREEMPTIVE || deprecation_level == DEP_LEVEL::FOR_REMOVAL) {
1235  skip_spaces();
1236  deprecation_version = std::max(deprecation_version, version_info(read_word()));
1237  }
1238  skip_spaces();
1239  if(!deprecation_detail.empty()){
1240  deprecation_detail += '\n';
1241  }
1242  deprecation_detail = read_rest_of_line();
1243  skip_eol();
1244  }
1245  }
1246 
1247  if(found_enddef > 0 && ++found_enddef == 7) {
1248  if(std::equal(buffer.end() - 6, buffer.end(), "enddef")) {
1249  break;
1250  } else {
1251  found_enddef = 0;
1252  if(std::equal(buffer.end() - 6, buffer.end(), "define")) { // TODO: Maybe add support for
1253  // this? This would fill feature
1254  // request #21343
1255  parent_.error(
1256  "Preprocessor error: #define is not allowed inside a #define/#enddef pair",
1257  linenum);
1258  }
1259  }
1260  }
1261  }
1262  }
1263 
1264  if(found_enddef != 7) {
1265  parent_.error("Unterminated preprocessor definition", linenum_);
1266  }
1267 
1268  if(!skipping_) {
1269  preproc_map::const_iterator old_i = parent_.defines_->find(symbol);
1270  if(old_i != parent_.defines_->end()) {
1271  std::ostringstream new_pos, old_pos;
1272  const preproc_define& old_d = old_i->second;
1273 
1274  new_pos << linenum << ' ' << parent_.location_;
1275  old_pos << old_d.linenum << ' ' << old_d.location;
1276 
1277  WRN_PREPROC << "Redefining macro " << symbol << " without explicit #undef at "
1278  << lineno_string(new_pos.str()) << '\n'
1279  << "previously defined at " << lineno_string(old_pos.str()) << '\n';
1280  }
1281 
1282  buffer.erase(buffer.end() - 7, buffer.end());
1283  (*parent_.defines_)[symbol]
1284  = preproc_define(buffer, items, optargs, parent_.textdomain_, linenum, parent_.location_,
1285  deprecation_detail, deprecation_level, deprecation_version);
1286 
1287  LOG_PREPROC << "defining macro " << symbol << " (location " << get_location(parent_.location_) << ")\n";
1288  }
1289  } else if(command == "ifdef") {
1290  skip_spaces();
1291  const std::string& symbol = read_word();
1292  bool found = parent_.defines_->count(symbol) != 0;
1293  DBG_PREPROC << "testing for macro " << symbol << ": " << (found ? "defined" : "not defined") << '\n';
1294  conditional_skip(!found);
1295  } else if(command == "ifndef") {
1296  skip_spaces();
1297  const std::string& symbol = read_word();
1298  bool found = parent_.defines_->count(symbol) != 0;
1299  DBG_PREPROC << "testing for macro " << symbol << ": " << (found ? "defined" : "not defined") << '\n';
1300  conditional_skip(found);
1301  } else if(command == "ifhave") {
1302  skip_spaces();
1303  const std::string& symbol = read_word();
1304  bool found = !filesystem::get_wml_location(symbol, directory_).empty();
1305  DBG_PREPROC << "testing for file or directory " << symbol << ": " << (found ? "found" : "not found")
1306  << '\n';
1307  conditional_skip(!found);
1308  } else if(command == "ifnhave") {
1309  skip_spaces();
1310  const std::string& symbol = read_word();
1311  bool found = !filesystem::get_wml_location(symbol, directory_).empty();
1312  DBG_PREPROC << "testing for file or directory " << symbol << ": " << (found ? "found" : "not found")
1313  << '\n';
1314  conditional_skip(found);
1315  } else if(command == "ifver" || command == "ifnver") {
1316  skip_spaces();
1317  const std::string& vsymstr = read_word();
1318  skip_spaces();
1319  const std::string& vopstr = read_word();
1320  skip_spaces();
1321  const std::string& vverstr = read_word();
1322 
1323  const VERSION_COMP_OP vop = parse_version_op(vopstr);
1324 
1325  if(vop == OP_INVALID) {
1326  parent_.error("Invalid #ifver/#ifnver operator", linenum_);
1327  } else if(parent_.defines_->count(vsymstr) != 0) {
1328  const preproc_define& sym = (*parent_.defines_)[vsymstr];
1329 
1330  if(!sym.arguments.empty()) {
1331  parent_.error("First argument macro in #ifver/#ifnver should not require arguments", linenum_);
1332  }
1333 
1334  version_info const version1(sym.value);
1335  version_info const version2(vverstr);
1336 
1337  const bool found = do_version_check(version1, vop, version2);
1338  DBG_PREPROC << "testing version '" << version1.str() << "' against '" << version2.str() << "' ("
1339  << vopstr << "): " << (found ? "match" : "no match") << '\n';
1340 
1341  conditional_skip(command == "ifver" ? !found : found);
1342  } else {
1343  std::string err = "Undefined macro in #ifver/#ifnver first argument: '";
1344  err += vsymstr;
1345  err += "'";
1346  parent_.error(err, linenum_);
1347  }
1348  } else if(command == "else") {
1349  if(token.type == token_desc::SKIP_ELSE) {
1350  pop_token();
1351  --skipping_;
1353  } else if(token.type == token_desc::PROCESS_IF) {
1354  pop_token();
1355  ++skipping_;
1357  } else {
1358  parent_.error("Unexpected #else", linenum_);
1359  }
1360  } else if(command == "endif") {
1361  switch(token.type) {
1362  case token_desc::SKIP_IF:
1363  case token_desc::SKIP_ELSE:
1364  --skipping_;
1367  break;
1368  default:
1369  parent_.error("Unexpected #endif", linenum_);
1370  }
1371  pop_token();
1372  } else if(command == "textdomain") {
1373  skip_spaces();
1374  const std::string& s = read_word();
1375  if(s != parent_.textdomain_) {
1376  put("#textdomain ");
1377  put(s);
1378  parent_.textdomain_ = s;
1379  }
1380  comment = true;
1381  } else if(command == "enddef") {
1382  parent_.error("Unexpected #enddef", linenum_);
1383  } else if(command == "undef") {
1384  skip_spaces();
1385  const std::string& symbol = read_word();
1386  if(!skipping_) {
1387  parent_.defines_->erase(symbol);
1388  LOG_PREPROC << "undefine macro " << symbol << " (location " << get_location(parent_.location_) << ")\n";
1389  }
1390  } else if(command == "error") {
1391  if(!skipping_) {
1392  skip_spaces();
1393  std::ostringstream error;
1394  error << "#error: \"" << read_rest_of_line() << '"';
1395  parent_.error(error.str(), linenum_);
1396  } else
1397  DBG_PREPROC << "Skipped an error\n";
1398  } else if(command == "warning") {
1399  if(!skipping_) {
1400  skip_spaces();
1401  std::ostringstream warning;
1402  warning << "#warning: \"" << read_rest_of_line() << '"';
1403  parent_.warning(warning.str(), linenum_);
1404  } else {
1405  DBG_PREPROC << "Skipped a warning\n";
1406  }
1407  } else if(command == "deprecated") {
1408  // The current file is deprecated, so print a message
1409  skip_spaces();
1411  try {
1412  level = DEP_LEVEL(std::stoi(read_word()));
1413  } catch(const std::invalid_argument&) {
1414  // Meh, just fall back to the default of PREEMPTIVE...
1415  }
1417  if(level == DEP_LEVEL::PREEMPTIVE || level == DEP_LEVEL::FOR_REMOVAL) {
1418  skip_spaces();
1419  version = version_info(read_word());
1420  }
1421  skip_spaces();
1422  std::string detail = read_rest_of_line();
1423  deprecated_message(get_filename(parent_.location_), level, version, detail);
1424  } else {
1425  comment = token.type != token_desc::MACRO_SPACE;
1426  }
1427 
1428  skip_eol();
1429  if(comment) {
1430  put('\n');
1431  }
1432  } else if(token.type == token_desc::MACRO_SPACE || token.type == token_desc::MACRO_CHUNK) {
1433  if(c == '(') {
1434  // If a macro argument was started, it is implicitly ended.
1435  token.type = token_desc::MACRO_SPACE;
1437  } else if(utils::portable_isspace(c)) {
1438  // If a macro argument was started, it is implicitly ended.
1439  token.type = token_desc::MACRO_SPACE;
1440  } else if(c == '}') {
1441  --slowpath_;
1442  if(skipping_) {
1443  pop_token();
1444  return true;
1445  }
1446 
1447  // FIXME: is this obsolete?
1448  // if (token.type == token_desc::MACRO_SPACE) {
1449  // if (!strings_.back().empty()) {
1450  // std::ostringstream error;
1451  // std::ostringstream location;
1452  // error << "Can't parse new macro parameter with a macro call scope open";
1453  // location<<linenum_<<' '<<parent_.location_;
1454  // parent_.error(error.str(), location.str());
1455  // }
1456  // strings_.pop_back();
1457  //}
1458 
1459  if(strings_.size() <= static_cast<std::size_t>(token.stack_pos)) {
1460  parent_.error("No macro or file substitution target specified", linenum_);
1461  }
1462 
1463  std::string symbol = strings_[token.stack_pos];
1464  std::string::size_type pos;
1465  while((pos = symbol.find(OUTPUT_SEPARATOR)) != std::string::npos) {
1466  std::string::iterator b = symbol.begin(); // invalidated at each iteration
1467  symbol.erase(b + pos, b + symbol.find('\n', pos + 1) + 1);
1468  }
1469 
1470  std::map<std::string, std::string>::const_iterator arg;
1471  preproc_map::const_iterator macro;
1472 
1473  // If this is a known pre-processing symbol, then we insert it,
1474  // otherwise we assume it's a file name to load.
1475  if(symbol == current_file_str && strings_.size() - token.stack_pos == 1) {
1476  pop_token();
1478  } else if(symbol == current_dir_str && strings_.size() - token.stack_pos == 1) {
1479  pop_token();
1481  } else if(symbol == left_curly_str && strings_.size() - token.stack_pos == 1) {
1482  pop_token();
1483  put("{");
1484  } else if(symbol == right_curly_str && strings_.size() - token.stack_pos == 1) {
1485  pop_token();
1486  put("}");
1487  } else if(local_defines_ && (arg = local_defines_->find(symbol)) != local_defines_->end()) {
1488  if(strings_.size() - token.stack_pos != 1) {
1489  std::ostringstream error;
1490  error << "Macro argument '" << symbol << "' does not expect any arguments";
1491  parent_.error(error.str(), linenum_);
1492  }
1493 
1494  std::ostringstream v;
1495  v << arg->second << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1496  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1497 
1498  pop_token();
1499  put(v.str());
1500  } else if(parent_.depth() < 100 && (macro = parent_.defines_->find(symbol)) != parent_.defines_->end()) {
1501  const preproc_define& val = macro->second;
1502  std::size_t nb_arg = strings_.size() - token.stack_pos - 1;
1503  std::size_t optional_arg_num = 0;
1504 
1505  std::unique_ptr<std::map<std::string, std::string>> defines{new std::map<std::string, std::string>};
1506  const std::string& dir = filesystem::directory_name(val.location.substr(0, val.location.find(' ')));
1507 
1508  if(val.is_deprecated()) {
1510  }
1511 
1512  for(std::size_t i = 0; i < nb_arg; ++i) {
1513  if(i < val.arguments.size()) {
1514  // Normal mandatory arguments
1515 
1516  (*defines)[val.arguments[i]] = strings_[token.stack_pos + i + 1];
1517  } else {
1518  // These should be optional argument overrides
1519 
1520  std::string str = strings_[token.stack_pos + i + 1];
1521  std::size_t equals_pos = str.find_first_of("=");
1522 
1523  if(equals_pos != std::string::npos) {
1524  std::size_t argname_pos = str.substr(0, equals_pos).find_last_of(" \n") + 1;
1525 
1526  std::string argname = str.substr(argname_pos, equals_pos - argname_pos);
1527 
1528  if(val.optional_arguments.find(argname) != val.optional_arguments.end()) {
1529  (*defines)[argname] = str.substr(equals_pos + 1);
1530 
1531  optional_arg_num++;
1532 
1533  DBG_PREPROC << "Found override for " << argname << " in call to macro " << symbol
1534  << "\n";
1535  } else {
1536  std::ostringstream warning;
1537  warning << "Unrecognized optional argument passed to macro '" << symbol << "': '"
1538  << argname << "'";
1539  parent_.warning(warning.str(), linenum_);
1540 
1541  optional_arg_num++; // To prevent the argument number check from blowing up
1542  }
1543  }
1544  }
1545  }
1546 
1547  // If the macro definition has any optional arguments, insert their defaults
1548  if(val.optional_arguments.size() > 0) {
1549  for(const auto& argument : val.optional_arguments) {
1550  if(defines->find(argument.first) == defines->end()) {
1551  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1552 
1554  std::istream in(buf.get());
1555 
1556  filesystem::scoped_istream buffer{new std::istringstream(argument.second)};
1557 
1558  std::unique_ptr<std::map<std::string, std::string>> temp_defines{new std::map<std::string, std::string>};
1559  temp_defines->insert(defines->begin(), defines->end());
1560 
1562  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(temp_defines), false);
1563 
1564  std::ostringstream res;
1565  res << in.rdbuf();
1566 
1567  DBG_PREPROC << "Setting default for optional argument " << argument.first << " in macro "
1568  << symbol << "\n";
1569 
1570  (*defines)[argument.first] = res.str();
1571  }
1572  }
1573  }
1574 
1575  if(nb_arg - optional_arg_num != val.arguments.size()) {
1576  const std::vector<std::string>& locations = utils::quoted_split(val.location, ' ');
1577  const std::string filename = locations.empty() ? "<command-line>" : get_filename(locations[0]);
1578  std::ostringstream error;
1579  error << "Preprocessor symbol '" << symbol << "' defined at " << filename << ":"
1580  << val.linenum << " expects " << val.arguments.size() << " arguments, but has "
1581  << nb_arg - optional_arg_num << " arguments";
1582  parent_.error(error.str(), linenum_);
1583  }
1584 
1585  filesystem::scoped_istream buffer{new std::istringstream(val.value)};
1586 
1587  pop_token();
1588 
1589  if(!slowpath_) {
1590  DBG_PREPROC << "substituting macro " << symbol << '\n';
1591 
1593  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1594  } else {
1595  DBG_PREPROC << "substituting (slow) macro " << symbol << '\n';
1596 
1597  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1598 
1599  // Make the nested preprocessor_data responsible for
1600  // restoring our current textdomain if needed.
1602 
1603  std::ostringstream res;
1604  {
1605  std::istream in(buf.get());
1607  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1608 
1609  res << in.rdbuf();
1610  }
1611 
1612  put(res.str());
1613  }
1614  } else if(parent_.depth() < 40) {
1615  LOG_PREPROC << "Macro definition not found for " << symbol << " , attempting to open as file.\n";
1616  pop_token();
1617 
1618  std::string nfname = filesystem::get_wml_location(symbol, directory_);
1619  if(!nfname.empty()) {
1620  if(!slowpath_)
1621  // nfname.size() - symbol.size() gives you an index into nfname
1622  // This does not necessarily match the symbol though, as it can start with ~ or ./
1623  parent_.add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1624  else {
1625  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1626 
1627  std::ostringstream res;
1628  {
1629  std::istream in(buf.get());
1630  buf->add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1631 
1632  res << in.rdbuf();
1633  }
1634 
1635  put(res.str());
1636  }
1637  } else {
1638  std::ostringstream error;
1639  error << "Macro/file '" << symbol << "' is missing";
1640  parent_.error(error.str(), linenum_);
1641  }
1642  } else {
1643  parent_.error("Too many nested preprocessing inclusions", linenum_);
1644  }
1645  } else if(!skipping_) {
1646  if(token.type == token_desc::MACRO_SPACE) {
1647  std::ostringstream s;
1648  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1649  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1650 
1651  strings_.push_back(s.str());
1652  token.type = token_desc::MACRO_CHUNK;
1653  }
1654  put(c);
1655  }
1656  } else {
1657  put(c);
1658  }
1659 
1660  return true;
1661 }
1662 
1663 
1664 // ==================================================================================
1665 // PREPROCESSOR SCOPE HELPER
1666 // ==================================================================================
1667 
1668 struct preprocessor_scope_helper : std::basic_istream<char>
1669 {
1670  preprocessor_scope_helper(const std::string& fname, preproc_map* defines)
1671  : std::basic_istream<char>(nullptr)
1672  , buf_(nullptr)
1673  , local_defines_(nullptr)
1674  {
1675  //
1676  // If no defines were provided, we create a new local preproc_map and assign
1677  // it to defines temporarily. In this case, the map will be deleted once this
1678  // object is destroyed and defines will still be subsequently null.
1679  //
1680  if(!defines) {
1681  local_defines_.reset(new preproc_map);
1682  defines = local_defines_.get();
1683  }
1684 
1685  buf_.reset(new preprocessor_streambuf(defines));
1686 
1687  // Begin processing.
1688  buf_->add_preprocessor<preprocessor_file>(fname);
1689 
1690  //
1691  // TODO: not sure if this call is needed. Previously, this call was passed a
1692  // preprocessor_streambuf pointer and the std::basic_istream constructor was
1693  // called with its contents. However, at that point the preprocessing should
1694  // already have completed, meaning this call might be redundant. Not sure.
1695  //
1696  // - vultraz, 2017-08-31
1697  //
1698  init(buf_.get());
1699  }
1700 
1702  {
1703  clear(std::ios_base::goodbit);
1704  exceptions(std::ios_base::goodbit);
1705  rdbuf(nullptr);
1706  }
1707 
1708  std::unique_ptr<preprocessor_streambuf> buf_;
1709  std::unique_ptr<preproc_map> local_defines_;
1710 };
1711 
1712 
1713 // ==================================================================================
1714 // FREE-STANDING FUNCTIONS
1715 // ==================================================================================
1716 
1717 filesystem::scoped_istream preprocess_file(const std::string& fname, preproc_map* defines)
1718 {
1719  log_scope("preprocessing file " + fname + " ...");
1720 
1721  // NOTE: the preprocessor_scope_helper does *not* take ownership of defines.
1722  return filesystem::scoped_istream(new preprocessor_scope_helper(fname, defines));
1723 }
1724 
1725 void preprocess_resource(const std::string& res_name,
1726  preproc_map* defines_map,
1727  bool write_cfg,
1728  bool write_plain_cfg,
1729  const std::string& parent_directory)
1730 {
1731  if(filesystem::is_directory(res_name)) {
1732  std::vector<std::string> dirs, files;
1733 
1736 
1737  // Subdirectories
1738  for(const std::string& dir : dirs) {
1739  LOG_PREPROC << "processing sub-dir: " << dir << '\n';
1740  preprocess_resource(dir, defines_map, write_cfg, write_plain_cfg, parent_directory);
1741  }
1742 
1743  // Files in current directory
1744  for(const std::string& file : files) {
1745  preprocess_resource(file, defines_map, write_cfg, write_plain_cfg, parent_directory);
1746  }
1747 
1748  return;
1749  }
1750 
1751  // process only config files.
1752  if(!filesystem::ends_with(res_name, ".cfg")) {
1753  return;
1754  }
1755 
1756  LOG_PREPROC << "processing resource: " << res_name << '\n';
1757 
1758  // disable filename encoding to get clear #line in cfg.plain
1759  encode_filename = false;
1760 
1761  filesystem::scoped_istream stream = preprocess_file(res_name, defines_map);
1762 
1763  std::stringstream ss;
1764 
1765  // Set the failbit so if we get any preprocessor exceptions (e.g.:preproc_config::error)
1766  // they will be propagated in the main program, instead of just setting the
1767  // failbit on the stream. This was necessary in order for the MSVC and GCC
1768  // binaries to behave the same way.
1769  ss.exceptions(std::ios_base::failbit);
1770 
1771  ss << (*stream).rdbuf();
1772 
1773  LOG_PREPROC << "processing finished\n";
1774 
1775  if(write_cfg || write_plain_cfg) {
1776  config cfg;
1777  std::string streamContent = ss.str();
1778 
1779  read(cfg, streamContent);
1780 
1781  const std::string preproc_res_name = parent_directory + "/" + filesystem::base_name(res_name);
1782 
1783  // Write the processed cfg file
1784  if(write_cfg) {
1785  LOG_PREPROC << "writing cfg file: " << preproc_res_name << '\n';
1786 
1788  filesystem::scoped_ostream outStream(filesystem::ostream_file(preproc_res_name));
1789 
1790  write(*outStream, cfg);
1791  }
1792 
1793  // Write the plain cfg file
1794  if(write_plain_cfg) {
1795  LOG_PREPROC << "writing plain cfg file: " << (preproc_res_name + ".plain") << '\n';
1796 
1798  filesystem::write_file(preproc_res_name + ".plain", streamContent);
1799  }
1800  }
1801 }
std::string lineno_string(const std::string &lineno)
VERSION_COMP_OP
static const char OUTPUT_SEPARATOR
std::vector< std::string > arguments
preprocessor * current() const
preprocessor_file(preprocessor_streambuf &t, const std::string &name, std::size_t symbol_index=-1)
Constructor.
void write(config_writer &, const std::string &) const
std::vector< std::string > files_
virtual bool get_chunk() override
Inserts and processes the next file in the list of included files.
preprocessor_scope_helper(const std::string &fname, preproc_map *defines)
Specialized preprocessor for handling a file or a set of files.
void read(const config &)
Interfaces for manipulating version numbers of engine, add-ons, etc.
static const std::string current_file_str
static int read_line(lua_State *L, FILE *f, int chop)
Definition: liolib.cpp:482
std::vector< std::string >::const_iterator end_
VERSION_COMP_OP parse_version_op(const std::string &op_str)
std::string read_word()
preproc_map default_defines_
static l_noret error(LoadState *S, const char *why)
Definition: lundump.cpp:39
buffered_istream in_
Input stream.
static std::map< std::string, int > file_number_map
bool ends_with(const std::string &str, const std::string &suffix)
#define ERR_PREPROC
child_itors child_range(config_key_type key)
Definition: config.cpp:362
virtual MODE parse_mode()
Returns the appropriate parsing mode for this preprocessor.
int peek()
Gets a character from the buffer.
Specialized preprocessor for handling any kind of input stream.
std::string out_buffer_
Buffer read by the STL stream.
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
Definition: filesystem.cpp:992
void clear(const std::string &key)
Definition: general.cpp:205
Helper class for buffering a std::istream.
virtual bool get_chunk() override
Preprocesses and sends some text to the parent_ buffer.
void preprocess_resource(const std::string &res_name, preproc_map *defines_map, bool write_cfg, bool write_plain_cfg, const std::string &parent_directory)
STL namespace.
void read_argument(const config &)
const std::vector< std::string > items
#define d
std::unique_ptr< preprocessor_streambuf > buf_
bool do_version_check(const version_info &a, VERSION_COMP_OP op, const version_info &b)
Definitions for the interface to Wesnoth Markup Language (WML).
std::vector< std::string > split(const std::string &val, const char c, const int flags)
Splits a (comma-)separated string into a vector of pieces.
std::vector< std::string >::const_iterator pos_
std::string location
Description of a preprocessing chunk.
#define WRN_PREPROC
#define b
void write_argument(config_writer &, const std::string &) const
unsigned in
If equal to search_counter, the node is off the list.
void write(std::ostream &out, const configr_of &cfg, unsigned int level)
Definition: parser.cpp:762
int slowpath_
Set to true whenever input tokens cannot be directly sent to the target buffer.
bool is_define_
True iff we are currently parsing a macros content, otherwise false.
void write_key_val(const std::string &key, const T &value)
This template function will work with any type that can be assigned to an attribute_value.
int get()
Gets and consumes a character from the buffer.
void conditional_skip(bool skip)
std::string deprecated_message(const std::string &elem_name, DEP_LEVEL level, const version_info &version, const std::string &detail)
Definition: deprecation.cpp:29
bool quoted_
Set to true if one preprocessor for this target started to read a string.
void close_child(const std::string &key)
void add_preprocessor(A &&... args)
static lg::log_domain log_preprocessor("preprocessor")
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:625
Class for writing a config out to a file in pieces.
version_info deprecation_version
void restore_old_preprocessor()
Restores the old preprocessing context.
std::string read_line()
void write_file(const std::string &fname, const std::string &data)
Throws io_exception if an error occurs.
int skipping_
Non-zero when the preprocessor has to skip some input text.
void push_token(token_desc::TOKEN_TYPE)
void open_child(const std::string &key)
std::deque< std::unique_ptr< preprocessor > > preprocessor_queue_
Input preprocessor queue.
virtual ~preprocessor()
bool is_deprecated() const
bool is_directory(const std::string &fname)
Returns true if the given file is a directory.
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:39
virtual void init() override
Allows specifying any actions that need to be called after the constructor completes.
std::string get_short_wml_path(const std::string &filename)
Returns a short path to filename, skipping the (user) data directory.
std::string deprecation_message
#define DBG_PREPROC
std::string value
#define LOG_PREPROC
std::string escape(const std::string &str, const char *special_chars)
Prepends a configurable set of characters with a backslash.
#define PACKAGE
Definition: wesconfig.h:23
std::string directory_
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:40
virtual preprocessor::MODE parse_mode() override
Returns the appropriate parsing mode for this preprocessor.
static std::string get_file_code(const std::string &filename)
Some defines: VERSION, PACKAGE, MIN_SAVEGAME_VERSION.
virtual void init()
Allows specifying any actions that need to be called after the constructor completes.
bool operator==(const preproc_define &) const
token_desc(TOKEN_TYPE type, const int stack_pos, const int linenum)
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, file_name_option mode, file_filter_option filter, file_reorder_option reorder, file_tree_checksum *checksum)
Populates &#39;files&#39; with all the files and &#39;dirs&#39; with all the directories in dir.
Definition: filesystem.cpp:352
boost::optional< DEP_LEVEL > deprecation_level
std::size_t i
Definition: function.cpp:933
logger & err()
Definition: log.cpp:78
std::string get_wml_location(const std::string &filename, const std::string &current_dir)
Returns a complete path to the actual WML file or directory or an empty string if the file isn&#39;t pres...
std::string read_rest_of_line()
std::unique_ptr< preproc_map > local_defines_
bool operator!=(preprocessor_data::token_desc::TOKEN_TYPE rhs, char lhs)
mock_party p
static std::string get_location(const std::string &loc)
static map_location::DIRECTION s
filesystem::scoped_istream in_scope_
Manages the lifetime of the std::istream pointer we own.
std::vector< std::string > quoted_split(const std::string &val, char c, int flags, char quote)
This function is identical to split(), except it does not split when it otherwise would if the previo...
friend class preprocessor_streambuf
#define log_scope(description)
Definition: log.hpp:186
Helper class for buffering a std::istream.
std::unique_ptr< std::map< std::string, std::string > > local_defines_
Mapping of macro arguments to their content.
Declarations for File-IO.
static int writer(lua_State *L, const void *b, size_t size, void *B)
Definition: lstrlib.cpp:182
void warning(const std::string &, int)
const version_info wesnoth_version(VERSION)
Base class for preprocessing an input.
std::vector< std::string > strings_
Buffer for delayed input processing.
Represents version numbers.
std::ostream & operator<<(std::ostream &stream, const preproc_define &def)
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
preprocessor_streambuf(const preprocessor_streambuf &t)
static std::string get_filename(const std::string &file_code)
double t
Definition: astarsearch.cpp:64
DEP_LEVEL
See https://wiki.wesnoth.org/CompatibilityStandards for more info.
Definition: deprecation.hpp:19
std::vector< token_desc > tokens_
Stack of nested preprocessing chunks.
static const std::string current_dir_str
preprocessor_streambuf & parent_
Standard logging facilities (interface).
std::string textdomain
std::string str() const
Serializes the version number into string form.
virtual int underflow() override
Inherited from basic_streambuf.
virtual bool get_chunk()=0
Preprocesses and sends some text to the parent_ buffer.
int stack_pos
Starting position in strings_ of the delayed text for this chunk.
preprocessor_data(preprocessor_streambuf &, filesystem::scoped_istream, const std::string &history, const std::string &name, int line, const std::string &dir, const std::string &domain, std::unique_ptr< std::map< std::string, std::string >> defines, bool is_define=false)
void error(const std::string &, int)
Target for sending preprocessed output.
std::map< std::string, struct preproc_define > preproc_map
#define e
static const std::string right_curly_str
bool portable_isspace(const char c)
preprocessor_streambuf(preproc_map *def)
bool operator==(preprocessor_data::token_desc::TOKEN_TYPE, char)
std::string get_current_file()
Decodes the filenames placed in a location.
std::string old_location_
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:68
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
static preproc_map::value_type read_pair(const config &)
mock_char c
const std::string & name_
std::string old_textdomain_
static map_location::DIRECTION n
bool operator<(const preproc_define &) const
static bool encode_filename
std::string::const_iterator iterator
Definition: tokenizer.hpp:24
std::stringstream buffer_
Buffer filled by the #current_ preprocessor.
preprocessor(preprocessor_streambuf &t)
Sets up a new preprocessor for stream buffer t.
std::string directory_name(const std::string &file)
Returns the directory name of a file, with filename stripped.
bool eof() const
Is the end of input reached?
static const std::string left_curly_str
static bool create_directory_if_missing_recursive(const bfs::path &dirpath)
Definition: filesystem.cpp:324
static std::string preprocessor_error_detail_prefix
filesystem::scoped_ostream ostream_file(const std::string &fname, bool create_directory)
std::map< std::string, std::string > optional_arguments