The Battle for Wesnoth  1.15.12+dev
preprocessor.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2003 by David White <dave@whitevine.net>
3  Copyright (C) 2005 - 2018 by Guillaume Melquiond <guillaume.melquiond@gmail.com>
4  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation; either version 2 of the License, or
9  (at your option) any later version.
10  This program is distributed in the hope that it will be useful,
11  but WITHOUT ANY WARRANTY.
12 
13  See the COPYING file for more details.
14 */
15 
16 /**
17  * @file
18  * WML preprocessor.
19  */
20 
22 
23 #include "buffered_istream.hpp"
24 #include "config.hpp"
25 #include "filesystem.hpp"
26 #include "log.hpp"
28 #include "serialization/parser.hpp"
30 #include "game_version.hpp"
31 #include "wesconfig.h"
32 #include "deprecation.hpp"
33 
34 #include <stdexcept>
35 #include <deque>
36 
37 static lg::log_domain log_preprocessor("preprocessor");
38 #define ERR_PREPROC LOG_STREAM(err, log_preprocessor)
39 #define WRN_PREPROC LOG_STREAM(warn, log_preprocessor)
40 #define LOG_PREPROC LOG_STREAM(info, log_preprocessor)
41 #define DBG_PREPROC LOG_STREAM(debug, log_preprocessor)
42 
43 static const std::string current_file_str = "CURRENT_FILE";
44 static const std::string current_dir_str = "CURRENT_DIRECTORY";
45 static const std::string left_curly_str = "LEFT_BRACE";
46 static const std::string right_curly_str = "RIGHT_BRACE";
47 
48 // map associating each filename encountered to a number
49 static std::map<std::string, int> file_number_map;
50 
51 static bool encode_filename = true;
52 
53 static std::string preprocessor_error_detail_prefix = "\n ";
54 
55 static const char OUTPUT_SEPARATOR = '\xFE';
56 
57 // get filename associated to this code
58 static std::string get_filename(const std::string& file_code)
59 {
60  if(!encode_filename) {
61  return file_code;
62  }
63 
64  std::stringstream s;
65  s << file_code;
66  int n = 0;
67  s >> std::hex >> n;
68 
69  for(const auto& p : file_number_map) {
70  if(p.second == n) {
71  return p.first;
72  }
73  }
74 
75  return "<unknown>";
76 }
77 
78 // Get code associated to this filename
79 static std::string get_file_code(const std::string& filename)
80 {
81  if(!encode_filename) {
82  return filename;
83  }
84 
85  // Current number of encountered filenames
86  static int current_file_number = 0;
87 
88  int& fnum = file_number_map[utils::escape(filename, " \\")];
89  if(fnum == 0) {
90  fnum = ++current_file_number;
91  }
92 
93  std::ostringstream shex;
94  shex << std::hex << fnum;
95 
96  return shex.str();
97 }
98 
99 // decode the filenames placed in a location
100 static std::string get_location(const std::string& loc)
101 {
102  std::string res;
103  std::vector<std::string> pos = utils::quoted_split(loc, ' ');
104 
105  if(pos.empty()) {
106  return res;
107  }
108 
109  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
110  while(true) {
111  res += get_filename(*(i++));
112 
113  if(i == end) {
114  break;
115  }
116 
117  res += ' ';
118  res += *(i++);
119 
120  if(i == end) {
121  break;
122  }
123 
124  res += ' ';
125  }
126 
127  return res;
128 }
129 
130 
131 // ==================================================================================
132 // PREPROC_DEFINE IMPLEMENTATION
133 // ==================================================================================
134 
136 {
137  return value == v.value && arguments == v.arguments;
138 }
139 
141 {
142  if(location < v.location) {
143  return true;
144  }
145 
146  if(v.location < location) {
147  return false;
148  }
149 
150  return linenum < v.linenum;
151 }
152 
153 void preproc_define::write_argument(config_writer& writer, const std::string& arg) const
154 {
155  const std::string key = "argument";
156 
157  writer.open_child(key);
158 
159  writer.write_key_val("name", arg);
160  writer.close_child(key);
161 }
162 
163 void preproc_define::write(config_writer& writer, const std::string& name) const
164 {
165  const std::string key = "preproc_define";
166  writer.open_child(key);
167 
168  writer.write_key_val("name", name);
169  writer.write_key_val("value", value);
170  writer.write_key_val("textdomain", textdomain);
171  writer.write_key_val("linenum", std::to_string(linenum));
172  writer.write_key_val("location", get_location(location));
173 
174  for(const std::string& arg : arguments) {
175  write_argument(writer, arg);
176  }
177 
178  writer.close_child(key);
179 }
180 
182 {
183  arguments.push_back(cfg["name"]);
184 }
185 
186 void preproc_define::read(const config& cfg)
187 {
188  value = cfg["value"].str();
189  textdomain = cfg["textdomain"].str();
190  linenum = cfg["linenum"];
191  location = cfg["location"].str();
192 
193  for(const config& arg : cfg.child_range("argument")) {
194  read_argument(arg);
195  }
196 }
197 
198 preproc_map::value_type preproc_define::read_pair(const config& cfg)
199 {
200  preproc_define second;
201  second.read(cfg);
202 
203  return preproc_map::value_type(cfg["name"], second);
204 }
205 
206 std::ostream& operator<<(std::ostream& stream, const preproc_define& def)
207 {
208  return stream << "value: " << def.value << " arguments: " << def.location;
209 }
210 
211 std::ostream& operator<<(std::ostream& stream, const preproc_map::value_type& def)
212 {
213  return stream << def.second;
214 }
215 
216 // ==================================================================================
217 // PREPROCESSOR BASE
218 // ==================================================================================
219 
221 
222 /**
223  * Base class for preprocessing an input.
224  */
226 {
228 
229 protected:
230  /**
231  * Sets up a new preprocessor for stream buffer \a t.
232  * Saves the current preprocessing context of #parent_. It will be automatically restored on destruction.
233  *
234  * It relies on preprocessor_streambuf so it's implemented after that class is declared.
235  */
237 
239 
240 public:
241  virtual ~preprocessor()
242  {
243  }
244 
245  /** Allows specifying any actions that need to be called after the constructor completes. */
246  virtual void init()
247  {
248  }
249 
250  /**
251  * Preprocesses and sends some text to the #parent_ buffer.
252  * @return false when the input has no data left.
253  */
254  virtual bool get_chunk() = 0;
255 
256  enum MODE { NO_PARSING, PARSES_FILE, PARSES_MACRO };
257 
258  /** Returns the appropriate parsing mode for this preprocessor. */
259  virtual MODE parse_mode()
260  {
261  return NO_PARSING;
262  }
263 
264 private:
265  std::string old_textdomain_;
266  std::string old_location_;
267 
269 };
270 
271 
272 // ==================================================================================
273 // PREPROCESSOR BUFFER
274 // ==================================================================================
275 
276 /**
277  * Target for sending preprocessed output.
278  * Objects of this class can be plugged into an STL stream.
279  */
280 class preprocessor_streambuf : public std::streambuf
281 {
282 public:
284  : std::streambuf()
285  , out_buffer_("")
286  , buffer_()
287  , preprocessor_queue_()
288  , defines_(def)
289  , default_defines_()
290  , textdomain_(PACKAGE)
291  , location_("")
292  , linenum_(0)
293  , quoted_(false)
294  {
295  }
296 
297  /** Decodes the filenames placed in a location. */
298  std::string get_current_file();
299 
300  void error(const std::string&, int);
301  void warning(const std::string&, int);
302 
303  template<typename T, typename... A>
304  void add_preprocessor(A&&... args)
305  {
306  preprocessor_queue_.emplace_back(new T(*this, std::forward<A>(args)...));
307  preprocessor_queue_.back()->init();
308  }
309 
311  {
312  preprocessor_queue_.pop_back();
313  }
314 
315  int depth() const
316  {
317  return preprocessor_queue_.size();
318  }
319 
321  {
322  return preprocessor_queue_.empty() ? nullptr : preprocessor_queue_.back().get();
323  }
324 
325 private:
327  : std::streambuf()
328  , out_buffer_("")
329  , buffer_()
330  , preprocessor_queue_()
331  , defines_(t.defines_)
332  , default_defines_()
333  , textdomain_(PACKAGE)
334  , location_("")
335  , linenum_(0)
336  , quoted_(t.quoted_)
337  {
338  }
339 
340  /** Inherited from basic_streambuf. */
341  virtual int underflow() override;
342 
343  void restore_old_preprocessor();
344 
345  /** Buffer read by the STL stream. */
346  std::string out_buffer_;
347 
348  /** Buffer filled by the _current_ preprocessor. */
349  std::stringstream buffer_;
350 
351  /** Input preprocessor queue. */
352  std::deque<std::unique_ptr<preprocessor>> preprocessor_queue_;
353 
356 
357  std::string textdomain_;
358  std::string location_;
359 
360  int linenum_;
361 
362  /**
363  * Set to true if one preprocessor for this target started to read a string.
364  * Deeper-nested preprocessors are then forbidden to.
365  */
366  bool quoted_;
367 
368  friend class preprocessor;
369  friend class preprocessor_file;
370  friend class preprocessor_data;
372 };
373 
374 /** Preprocessor constructor. */
376  : parent_(t)
377  , old_textdomain_(t.textdomain_)
378  , old_location_(t.location_)
379  , old_linenum_(t.linenum_)
380 {
381 }
382 
383 /**
384  * Called by an STL stream whenever it has reached the end of #out_buffer_.
385  * Fills #buffer_ by calling the _current_ preprocessor, then copies its
386  * content into #out_buffer_.
387  * @return the first character of #out_buffer_ if any, EOF otherwise.
388  */
390 {
391  unsigned sz = 0;
392  if(char* gp = gptr()) {
393  if(gp < egptr()) {
394  // Sanity check: the internal buffer has not been totally consumed,
395  // should we force the caller to use what remains first?
396  return *gp;
397  }
398 
399  // The buffer has been completely read; fill it again.
400  // Keep part of the previous buffer, to ensure putback capabilities.
401  sz = out_buffer_.size();
402  buffer_.str(std::string());
403 
404  if(sz > 3) {
405  buffer_ << out_buffer_.substr(sz - 3);
406  sz = 3;
407  } else {
408  buffer_ << out_buffer_;
409  }
410  } else {
411  // The internal get-data pointer is null
412  }
413 
414  const int desired_fill_amount = 2000;
415 
416  while(current() && buffer_.rdbuf()->in_avail() < desired_fill_amount) {
417  // Process files and data chunks until the desired buffer size is reached
418  if(!current()->get_chunk()) {
419  // Drop the current preprocessor item from the queue.
420  restore_old_preprocessor();
421  }
422  }
423 
424  // Update the internal state and data pointers
425  out_buffer_ = buffer_.str();
426  if(out_buffer_.empty()) {
427  return EOF;
428  }
429 
430  char* begin = &*out_buffer_.begin();
431  unsigned bs = out_buffer_.size();
432 
433  setg(begin, begin + sz, begin + bs);
434 
435  if(sz >= bs) {
436  return EOF;
437  }
438 
439  return static_cast<unsigned char>(*(begin + sz));
440 }
441 
442 /**
443 * Restores the old preprocessing context.
444 * Appends location and domain directives to the buffer, so that the parser
445 * notices these changes.
446 */
448 {
449  preprocessor* current = this->current();
450 
451  if(!current->old_location_.empty()) {
452  buffer_ << OUTPUT_SEPARATOR << "line " << current->old_linenum_ << ' ' << current->old_location_ << '\n';
453  }
454 
455  if(!current->old_textdomain_.empty() && textdomain_ != current->old_textdomain_) {
456  buffer_ << OUTPUT_SEPARATOR << "textdomain " << current->old_textdomain_ << '\n';
457  }
458 
459  location_ = current->old_location_;
460  linenum_ = current->old_linenum_;
461  textdomain_ = current->old_textdomain_;
462 
463  // Drop the preprocessor from the queue.
464  drop_preprocessor();
465 }
466 
468 {
469  unsigned nested_level = 0;
470 
471  preprocessor* pre = nullptr;
472 
473  // Iterate backwards over queue to get the last non-macro preprocessor.
474  for(auto p = preprocessor_queue_.rbegin(); p != preprocessor_queue_.rend(); ++p) {
475  pre = p->get();
476 
477  if(!pre || pre->parse_mode() == preprocessor::PARSES_FILE) {
478  break;
479  }
480 
481  if(pre->parse_mode() == preprocessor::PARSES_MACRO) {
482  ++nested_level;
483  }
484  }
485 
486  std::string res;
487  std::vector<std::string> pos = utils::quoted_split(location_, ' ');
488 
489  if(pos.size() <= 2 * nested_level) {
490  return res;
491  }
492 
493  return get_filename(pos[2 * nested_level]);
494 }
495 
496 std::string lineno_string(const std::string& lineno)
497 {
498  std::vector<std::string> pos = utils::quoted_split(lineno, ' ');
499  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
500  std::string included_from = preprocessor_error_detail_prefix + "included from ";
501  std::string res;
502 
503  while(i != end) {
504  const std::string& line = *(i++);
505 
506  if(!res.empty()) {
507  res += included_from;
508  }
509 
510  if(i != end) {
511  res += get_filename(*(i++));
512  } else {
513  res += "<unknown>";
514  }
515 
516  res += ':' + line;
517  }
518 
519  if(res.empty()) {
520  res = "???";
521  }
522 
523  return res;
524 }
525 
526 void preprocessor_streambuf::error(const std::string& error_type, int l)
527 {
528  std::string position, error;
529  std::ostringstream pos;
530 
531  pos << l << ' ' << location_;
532  position = lineno_string(pos.str());
533 
534  error = error_type + '\n';
535  error += "at " + position;
536 
537  ERR_PREPROC << error << '\n';
538 
539  throw preproc_config::error(error);
540 }
541 
542 void preprocessor_streambuf::warning(const std::string& warning_type, int l)
543 {
544  std::string position, warning;
545  std::ostringstream pos;
546 
547  pos << l << ' ' << location_;
548  position = lineno_string(pos.str());
549 
550  warning = warning_type + '\n';
551  warning += "at " + position;
552 
553  WRN_PREPROC << warning << '\n';
554 }
555 
556 
557 // ==================================================================================
558 // PREPROCESSOR FILE
559 // ==================================================================================
560 
561 /**
562  * Specialized preprocessor for handling a file or a set of files.
563  * A preprocessor_file object is created when a preprocessor encounters an
564  * inclusion directive that resolves to a file or directory, e.g. '{themes/}'.
565  */
567 {
568 public:
569  /** Constructor. It relies on preprocessor_data so it's implemented after that class is declared. */
570  preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index = -1);
571 
572  virtual void init() override;
573 
574  /**
575  * Inserts and processes the next file in the list of included files.
576  * @return false if there is no next file.
577  */
578  virtual bool get_chunk() override
579  {
580  while(pos_ != end_) {
581  const std::string& name = *(pos_++);
582  unsigned sz = name.size();
583 
584  // Use reverse iterator to optimize testing
585  if(sz < 5 || !std::equal(name.rbegin(), name.rbegin() + 4, "gfc.")) {
586  continue;
587  }
588 
590  return true;
591  }
592 
593  return false;
594  }
595 
596 private:
597  std::vector<std::string> files_;
598  std::vector<std::string>::const_iterator pos_, end_;
599 
600  const std::string& name_;
601 
603 };
604 
605 
606 // ==================================================================================
607 // PREPROCESSOR DATA
608 // ==================================================================================
609 
610 /**
611  * Specialized preprocessor for handling any kind of input stream.
612  * This is the core of the preprocessor.
613  */
615 {
616  /** Description of a preprocessing chunk. */
617  struct token_desc
618  {
619  enum TOKEN_TYPE {
620  START, // Toplevel
621  PROCESS_IF, // Processing the "if" branch of a ifdef/ifndef (the "else" branch will be skipped)
622  PROCESS_ELSE, // Processing the "else" branch of a ifdef/ifndef
623  SKIP_IF, // Skipping the "if" branch of a ifdef/ifndef (the "else" branch, if any, will be processed)
624  SKIP_ELSE, // Skipping the "else" branch of a ifdef/ifndef
625  STRING, // Processing a string
626  VERBATIM, // Processing a verbatim string
627  MACRO_SPACE, // Processing between chunks of a macro call (skip spaces)
628  MACRO_CHUNK, // Processing inside a chunk of a macro call (stop on space or '(')
629  MACRO_PARENS // Processing a parenthesized macro argument
630  };
631 
632  token_desc(TOKEN_TYPE type, const int stack_pos, const int linenum)
633  : type(type)
634  , stack_pos(stack_pos)
635  , linenum(linenum)
636  {
637  }
638 
640 
641  /** Starting position in #strings_ of the delayed text for this chunk. */
643  int linenum;
644  };
645 
646  /**
647  * Manages the lifetime of the @c std::istream pointer we own.
648  *
649  * Since @ref in_ uses the stream as well this object must be created
650  * before @ref in_ and destroyed after @ref in_ is destroyed.
651  */
653 
654  /** Input stream. */
656 
657  std::string directory_;
658 
659  /** Buffer for delayed input processing. */
660  std::vector<std::string> strings_;
661 
662  /** Mapping of macro arguments to their content. */
663  std::unique_ptr<std::map<std::string, std::string>> local_defines_;
664 
665  /** Stack of nested preprocessing chunks. */
666  std::vector<token_desc> tokens_;
667 
668  /**
669  * Set to true whenever input tokens cannot be directly sent to the target
670  * buffer. For instance, this happens with macro arguments. In that case,
671  * the output is redirected toward #strings_ until it can be processed.
672  */
674 
675  /**
676  * Non-zero when the preprocessor has to skip some input text.
677  * Increased whenever entering a conditional branch that is not useful,
678  * e.g. a ifdef that evaluates to false.
679  */
681  int linenum_;
682 
683  /** True iff we are currently parsing a macros content, otherwise false. */
685 
686  std::string read_word();
687  std::string read_line();
688  std::string read_rest_of_line();
689 
690  void skip_spaces();
691  void skip_eol();
692  void push_token(token_desc::TOKEN_TYPE);
693  void pop_token();
694  void put(char);
695  void put(const std::string& /*, int change_line = 0 */);
696  void conditional_skip(bool skip);
697 
698 public:
701  const std::string& history,
702  const std::string& name,
703  int line,
704  const std::string& dir,
705  const std::string& domain,
706  std::unique_ptr<std::map<std::string, std::string>> defines,
707  bool is_define = false);
708 
709  virtual bool get_chunk() override;
710 
711  virtual preprocessor::MODE parse_mode() override
712  {
713  return is_define_ ? PARSES_MACRO : PARSES_FILE;
714  }
715 
720 };
721 
723 {
724  throw std::logic_error("don't compare tokens with characters");
725 }
726 
728 {
729  return rhs == lhs;
730 }
731 
733 {
734  return !(lhs == rhs);
735 }
736 
738 {
739  return rhs != lhs;
740 }
741 
742 /** preprocessor_file constructor. */
743 preprocessor_file::preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index)
744  : preprocessor(t)
745  , files_()
746  , pos_()
747  , end_()
748  , name_(name)
749  , is_directory_(filesystem::is_directory(name))
750 {
751  if(is_directory_) {
752  filesystem::get_files_in_dir(name, &files_, nullptr,
756  );
757 
758  for(const std::string& fname : files_) {
759  std::size_t cpos = fname.rfind(" ");
760 
761  if(cpos != std::string::npos && cpos >= symbol_index) {
762  std::stringstream ss;
763  ss << "Found filename containing whitespace: '" << filesystem::base_name(fname)
764  << "' in included directory '" << name << "'.\nThe included symbol probably looks similar to '"
765  << filesystem::directory_name(fname.substr(symbol_index)) << "'";
766 
767  // TODO: find a real linenumber
768  parent_.error(ss.str(), -1);
769  }
770  }
771  } else {
772  // Handled in the init() function.
773  }
774 
775  pos_ = files_.begin();
776  end_ = files_.end();
777 }
778 
780 {
781  if(is_directory_) {
782  return;
783  }
784 
786 
787  if(!file_stream->good()) {
788  ERR_PREPROC << "Could not open file " << name_ << std::endl;
789  } else {
790  parent_.add_preprocessor<preprocessor_data>(std::move(file_stream), "",
793  );
794  }
795 }
796 
799  const std::string& history,
800  const std::string& name,
801  int linenum,
802  const std::string& directory,
803  const std::string& domain,
804  std::unique_ptr<std::map<std::string, std::string>> defines,
805  bool is_define)
806  : preprocessor(t)
807  , in_scope_(std::move(i))
808  , in_(*in_scope_)
809  , directory_(directory)
810  , strings_()
811  , local_defines_(std::move(defines))
812  , tokens_()
813  , slowpath_(0)
814  , skipping_(0)
815  , linenum_(linenum)
816  , is_define_(is_define)
817 {
818  std::ostringstream s;
819  s << history;
820 
821  if(!name.empty()) {
822  if(!history.empty()) {
823  s << ' ';
824  }
825 
826  s << get_file_code(name);
827  }
828 
829  if(!t.location_.empty()) {
830  s << ' ' << t.linenum_ << ' ' << t.location_;
831  }
832 
833  t.location_ = s.str();
834  t.linenum_ = linenum;
835 
836  t.buffer_ << OUTPUT_SEPARATOR << "line " << linenum << ' ' << t.location_ << '\n';
837 
838  if(t.textdomain_ != domain) {
839  t.buffer_ << OUTPUT_SEPARATOR << "textdomain " << domain << '\n';
840  t.textdomain_ = domain;
841  }
842 
844 }
845 
847 {
848  tokens_.emplace_back(t, strings_.size(), linenum_);
849 
850  if(t == token_desc::MACRO_SPACE) {
851  // Macro expansions do not have any associated storage at start.
852  return;
853  } else if(t == token_desc::STRING || t == token_desc::VERBATIM) {
854  /* Quoted strings are always inlined in the parent token. So
855  * they need neither storage nor metadata, unless the parent
856  * token is a macro expansion.
857  */
858  token_desc::TOKEN_TYPE& outer_type = tokens_[tokens_.size() - 2].type;
859  if(outer_type != token_desc::MACRO_SPACE) {
860  return;
861  }
862 
863  outer_type = token_desc::MACRO_CHUNK;
864  tokens_.back().stack_pos = strings_.size() + 1;
865  }
866 
867  std::ostringstream s;
868  if(!skipping_ && slowpath_) {
869  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
870  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
871  }
872 
873  strings_.push_back(s.str());
874 }
875 
877 {
878  token_desc::TOKEN_TYPE inner_type = tokens_.back().type;
879  unsigned stack_pos = tokens_.back().stack_pos;
880 
881  tokens_.pop_back();
882 
883  token_desc::TOKEN_TYPE& outer_type = tokens_.back().type;
884 
885  if(inner_type == token_desc::MACRO_PARENS) {
886  // Parenthesized macro arguments are left on the stack.
887  assert(outer_type == token_desc::MACRO_SPACE);
888  return;
889  }
890 
891  if(inner_type == token_desc::STRING || inner_type == token_desc::VERBATIM) {
892  // Quoted strings are always inlined.
893  assert(stack_pos == strings_.size());
894  return;
895  }
896 
897  if(outer_type == token_desc::MACRO_SPACE) {
898  /* A macro expansion does not have any associated storage.
899  * Instead, storage of the inner token is not discarded
900  * but kept as a new macro argument. But if the inner token
901  * was a macro expansion, it is about to be appended, so
902  * prepare for it.
903  */
904  if(inner_type == token_desc::MACRO_SPACE || inner_type == token_desc::MACRO_CHUNK) {
905  strings_.erase(strings_.begin() + stack_pos, strings_.end());
906  strings_.emplace_back();
907  }
908 
909  assert(stack_pos + 1 == strings_.size());
910  outer_type = token_desc::MACRO_CHUNK;
911 
912  return;
913  }
914 
915  strings_.erase(strings_.begin() + stack_pos, strings_.end());
916 }
917 
919 {
920  while(true) {
921  int c = in_.peek();
922 
923  if(in_.eof() || (c != ' ' && c != '\t')) {
924  return;
925  }
926 
927  in_.get();
928  }
929 }
930 
932 {
933  while(true) {
934  int c = in_.get();
935 
936  if(c == '\n') {
937  ++linenum_;
938  return;
939  }
940 
941  if(in_.eof()) {
942  return;
943  }
944  }
945 }
946 
948 {
949  std::string res;
950 
951  while(true) {
952  int c = in_.peek();
953 
954  if(c == preprocessor_streambuf::traits_type::eof() || utils::portable_isspace(c)) {
955  // DBG_PREPROC << "(" << res << ")\n";
956  return res;
957  }
958 
959  in_.get();
960  res += static_cast<char>(c);
961  }
962 }
963 
965 {
966  std::string res;
967 
968  while(true) {
969  int c = in_.get();
970 
971  if(c == '\n') {
972  ++linenum_;
973  return res;
974  }
975 
976  if(in_.eof()) {
977  return res;
978  }
979 
980  if(c != '\r') {
981  res += static_cast<char>(c);
982  }
983  }
984 }
985 
987 {
988  std::string res;
989 
990  while(in_.peek() != '\n' && !in_.eof()) {
991  int c = in_.get();
992 
993  if(c != '\r') {
994  res += static_cast<char>(c);
995  }
996  }
997 
998  return res;
999 }
1000 
1002 {
1003  if(skipping_) {
1004  return;
1005  }
1006 
1007  if(slowpath_) {
1008  strings_.back() += c;
1009  return;
1010  }
1011 
1012  int cond_linenum = c == '\n' ? linenum_ - 1 : linenum_;
1013 
1014  if(unsigned diff = cond_linenum - parent_.linenum_) {
1015  parent_.linenum_ = cond_linenum;
1016 
1017  if(diff <= parent_.location_.size() + 11) {
1018  parent_.buffer_ << std::string(diff, '\n');
1019  } else {
1020  parent_.buffer_ << OUTPUT_SEPARATOR << "line " << parent_.linenum_ << ' ' << parent_.location_ << '\n';
1021  }
1022  }
1023 
1024  if(c == '\n') {
1025  ++parent_.linenum_;
1026  }
1027 
1028  parent_.buffer_ << c;
1029 }
1030 
1031 void preprocessor_data::put(const std::string& s /*, int line_change*/)
1032 {
1033  if(skipping_) {
1034  return;
1035  }
1036 
1037  if(slowpath_) {
1038  strings_.back() += s;
1039  return;
1040  }
1041 
1042  parent_.buffer_ << s;
1043  // parent_.linenum_ += line_change;
1044 }
1045 
1047 {
1048  if(skip) {
1049  ++skipping_;
1050  }
1051 
1053 }
1054 
1056 {
1057  char c = static_cast<char>(in_.get());
1058  token_desc& token = tokens_.back();
1059 
1060  if(in_.eof()) {
1061  // The end of file was reached.
1062  // Make sure we don't have any incomplete tokens.
1063  char const* s;
1064 
1065  switch(token.type) {
1066  case token_desc::START:
1067  return false; // everything is fine
1069  case token_desc::SKIP_IF:
1071  case token_desc::SKIP_ELSE:
1072  s = "#ifdef or #ifndef";
1073  break;
1074  case token_desc::STRING:
1075  s = "Quoted string";
1076  break;
1077  case token_desc::VERBATIM:
1078  s = "Verbatim string";
1079  break;
1082  s = "Macro substitution";
1083  break;
1085  s = "Macro argument";
1086  break;
1087  default:
1088  s = "???";
1089  }
1090 
1091  parent_.error(std::string(s) + " not terminated", token.linenum);
1092  }
1093 
1094  if(c == '\n') {
1095  ++linenum_;
1096  }
1097 
1098  if(c == OUTPUT_SEPARATOR) {
1099  std::string buffer(1, c);
1100 
1101  while(true) {
1102  char d = static_cast<char>(in_.get());
1103 
1104  if(in_.eof() || d == '\n') {
1105  break;
1106  }
1107 
1108  buffer += d;
1109  }
1110 
1111  buffer += '\n';
1112  // line_change = 1-1 = 0
1113  put(buffer);
1114  } else if(token.type == token_desc::VERBATIM) {
1115  put(c);
1116 
1117  if(c == '>' && in_.peek() == '>') {
1118  put(in_.get());
1119  pop_token();
1120  }
1121  } else if(c == '<' && in_.peek() == '<') {
1122  in_.get();
1124  put('<');
1125  put('<');
1126  } else if(c == '"') {
1127  if(token.type == token_desc::STRING) {
1128  parent_.quoted_ = false;
1129  put(c);
1130  pop_token();
1131  } else if(!parent_.quoted_) {
1132  parent_.quoted_ = true;
1134  put(c);
1135  } else {
1136  parent_.error("Nested quoted string", linenum_);
1137  }
1138  } else if(c == '{') {
1140  ++slowpath_;
1141  } else if(c == ')' && token.type == token_desc::MACRO_PARENS) {
1142  pop_token();
1143  } else if(c == '#' && !parent_.quoted_) {
1144  std::string command = read_word();
1145  bool comment = false;
1146 
1147  if(command == "define") {
1148  skip_spaces();
1149  int linenum = linenum_;
1150  std::vector<std::string> items = utils::split(read_line(), ' ');
1151  std::map<std::string, std::string> optargs;
1152 
1153  if(items.empty()) {
1154  parent_.error("No macro name found after #define directive", linenum);
1155  }
1156 
1157  std::string symbol = items.front();
1158  items.erase(items.begin());
1159  int found_arg = 0, found_enddef = 0, found_deprecate = 0;
1160  std::optional<DEP_LEVEL> deprecation_level;
1161  std::string buffer, deprecation_detail;
1162  version_info deprecation_version = game_config::wesnoth_version;
1163  while(true) {
1164  if(in_.eof())
1165  break;
1166  char d = static_cast<char>(in_.get());
1167  if(d == '\n')
1168  ++linenum_;
1169  buffer += d;
1170  if(d == '#') {
1171  if(in_.peek() == 'a') {
1172  found_arg = 1;
1173  } else if(in_.peek() == 'd') {
1174  found_deprecate = 1;
1175  } else {
1176  found_enddef = 1;
1177  }
1178  } else {
1179  if(found_arg > 0 && ++found_arg == 4) {
1180  if(std::equal(buffer.end() - 3, buffer.end(), "arg")) {
1181  buffer.erase(buffer.end() - 4, buffer.end());
1182 
1183  skip_spaces();
1184  std::string argname = read_word();
1185  skip_eol();
1186 
1187  std::string argbuffer;
1188 
1189  int found_endarg = 0;
1190  while(true) {
1191  if(in_.eof()) {
1192  break;
1193  }
1194 
1195  char e = static_cast<char>(in_.get());
1196  if(e == '\n') {
1197  ++linenum_;
1198  }
1199 
1200  argbuffer += e;
1201 
1202  if(e == '#') {
1203  found_endarg = 1;
1204  } else if(found_endarg > 0 && ++found_endarg == 7) {
1205  if(std::equal(argbuffer.end() - 6, argbuffer.end(), "endarg")) {
1206  argbuffer.erase(argbuffer.end() - 7, argbuffer.end());
1207  optargs[argname] = argbuffer;
1208  skip_eol();
1209  break;
1210  } else {
1211  parent_.error("Unterminated #arg definition", linenum_);
1212  }
1213  }
1214  }
1215  }
1216  }
1217 
1218  if(found_deprecate > 0 && ++found_deprecate == 11) {
1219  if(std::equal(buffer.end() - 10, buffer.end(), "deprecated")) {
1220  buffer.erase(buffer.end() - 11, buffer.end());
1221  skip_spaces();
1222  try {
1223  DEP_LEVEL level = DEP_LEVEL(std::stoi(read_word()));
1224  if(deprecation_level) {
1225  deprecation_level = std::max(*deprecation_level, level);
1226  } else {
1227  deprecation_level = level;
1228  }
1229  } catch(const std::invalid_argument&) {
1230  // Meh, fall back to default of PREEMPTIVE...
1231  deprecation_level = DEP_LEVEL::PREEMPTIVE;
1232  }
1233  deprecation_version = game_config::wesnoth_version;
1234  if(deprecation_level == DEP_LEVEL::PREEMPTIVE || deprecation_level == DEP_LEVEL::FOR_REMOVAL) {
1235  skip_spaces();
1236  deprecation_version = std::max(deprecation_version, version_info(read_word()));
1237  }
1238  skip_spaces();
1239  if(!deprecation_detail.empty()){
1240  deprecation_detail += '\n';
1241  }
1242  deprecation_detail += read_rest_of_line();
1243  skip_eol();
1244  }
1245  }
1246 
1247  if(found_enddef > 0 && ++found_enddef == 7) {
1248  if(std::equal(buffer.end() - 6, buffer.end(), "enddef")) {
1249  break;
1250  } else {
1251  found_enddef = 0;
1252  if(std::equal(buffer.end() - 6, buffer.end(), "define")) { // TODO: Maybe add support for
1253  // this? This would fill feature
1254  // request #21343
1255  parent_.error(
1256  "Preprocessor error: #define is not allowed inside a #define/#enddef pair",
1257  linenum);
1258  }
1259  }
1260  }
1261  }
1262  }
1263 
1264  if(found_enddef != 7) {
1265  parent_.error("Unterminated preprocessor definition", linenum_);
1266  }
1267 
1268  if(!skipping_) {
1269  preproc_map::const_iterator old_i = parent_.defines_->find(symbol);
1270  if(old_i != parent_.defines_->end()) {
1271  std::ostringstream new_pos, old_pos;
1272  const preproc_define& old_d = old_i->second;
1273 
1274  new_pos << linenum << ' ' << parent_.location_;
1275  old_pos << old_d.linenum << ' ' << old_d.location;
1276 
1277  WRN_PREPROC << "Redefining macro " << symbol << " without explicit #undef at "
1278  << lineno_string(new_pos.str()) << '\n'
1279  << "previously defined at " << lineno_string(old_pos.str()) << '\n';
1280  }
1281 
1282  buffer.erase(buffer.end() - 7, buffer.end());
1283  (*parent_.defines_)[symbol]
1284  = preproc_define(buffer, items, optargs, parent_.textdomain_, linenum, parent_.location_,
1285  deprecation_detail, deprecation_level, deprecation_version);
1286 
1287  LOG_PREPROC << "defining macro " << symbol << " (location " << get_location(parent_.location_) << ")\n";
1288  }
1289  } else if(command == "ifdef" || command == "ifndef") {
1290  const bool negate = command[2] == 'n';
1291  skip_spaces();
1292  const std::string& symbol = read_word();
1293  bool found = parent_.defines_->count(symbol) != 0;
1294  DBG_PREPROC << "testing for macro " << symbol << ": " << (found ? "defined" : "not defined") << '\n';
1295  conditional_skip(negate ? found : !found);
1296  } else if(command == "ifhave" || command == "ifnhave") {
1297  const bool negate = command[2] == 'n';
1298  skip_spaces();
1299  const std::string& symbol = read_word();
1300  bool found = !filesystem::get_wml_location(symbol, directory_).empty();
1301  DBG_PREPROC << "testing for file or directory " << symbol << ": " << (found ? "found" : "not found")
1302  << '\n';
1303  conditional_skip(negate ? found : !found);
1304  } else if(command == "ifver" || command == "ifnver") {
1305  const bool negate = command[2] == 'n';
1306 
1307  skip_spaces();
1308  const std::string& vsymstr = read_word();
1309  skip_spaces();
1310  const std::string& vopstr = read_word();
1311  skip_spaces();
1312  const std::string& vverstr = read_word();
1313 
1314  const VERSION_COMP_OP vop = parse_version_op(vopstr);
1315 
1316  if(vop == OP_INVALID) {
1317  parent_.error("Invalid #ifver/#ifnver operator", linenum_);
1318  } else if(parent_.defines_->count(vsymstr) != 0) {
1319  const preproc_define& sym = (*parent_.defines_)[vsymstr];
1320 
1321  if(!sym.arguments.empty()) {
1322  parent_.error("First argument macro in #ifver/#ifnver should not require arguments", linenum_);
1323  }
1324 
1325  version_info const version1(sym.value);
1326  version_info const version2(vverstr);
1327 
1328  const bool found = do_version_check(version1, vop, version2);
1329  DBG_PREPROC << "testing version '" << version1.str() << "' against '" << version2.str() << "' ("
1330  << vopstr << "): " << (found ? "match" : "no match") << '\n';
1331 
1332  conditional_skip(negate ? found : !found);
1333  } else {
1334  std::string err = "Undefined macro in #ifver/#ifnver first argument: '";
1335  err += vsymstr;
1336  err += "'";
1337  parent_.error(err, linenum_);
1338  }
1339  } else if(command == "else") {
1340  if(token.type == token_desc::SKIP_ELSE) {
1341  pop_token();
1342  --skipping_;
1344  } else if(token.type == token_desc::PROCESS_IF) {
1345  pop_token();
1346  ++skipping_;
1348  } else {
1349  parent_.error("Unexpected #else", linenum_);
1350  }
1351  } else if(command == "endif") {
1352  switch(token.type) {
1353  case token_desc::SKIP_IF:
1354  case token_desc::SKIP_ELSE:
1355  --skipping_;
1358  break;
1359  default:
1360  parent_.error("Unexpected #endif", linenum_);
1361  }
1362  pop_token();
1363  } else if(command == "textdomain") {
1364  skip_spaces();
1365  const std::string& s = read_word();
1366  if(s != parent_.textdomain_) {
1367  put("#textdomain ");
1368  put(s);
1369  parent_.textdomain_ = s;
1370  }
1371  comment = true;
1372  } else if(command == "enddef") {
1373  parent_.error("Unexpected #enddef", linenum_);
1374  } else if(command == "undef") {
1375  skip_spaces();
1376  const std::string& symbol = read_word();
1377  if(!skipping_) {
1378  parent_.defines_->erase(symbol);
1379  LOG_PREPROC << "undefine macro " << symbol << " (location " << get_location(parent_.location_) << ")\n";
1380  }
1381  } else if(command == "error") {
1382  if(!skipping_) {
1383  skip_spaces();
1384  std::ostringstream error;
1385  error << "#error: \"" << read_rest_of_line() << '"';
1386  parent_.error(error.str(), linenum_);
1387  } else
1388  DBG_PREPROC << "Skipped an error\n";
1389  } else if(command == "warning") {
1390  if(!skipping_) {
1391  skip_spaces();
1392  std::ostringstream warning;
1393  warning << "#warning: \"" << read_rest_of_line() << '"';
1394  parent_.warning(warning.str(), linenum_);
1395  } else {
1396  DBG_PREPROC << "Skipped a warning\n";
1397  }
1398  } else if(command == "deprecated") {
1399  // The current file is deprecated, so print a message
1400  skip_spaces();
1402  try {
1403  level = DEP_LEVEL(std::stoi(read_word()));
1404  } catch(const std::invalid_argument&) {
1405  // Meh, just fall back to the default of PREEMPTIVE...
1406  }
1408  if(level == DEP_LEVEL::PREEMPTIVE || level == DEP_LEVEL::FOR_REMOVAL) {
1409  skip_spaces();
1410  version = version_info(read_word());
1411  }
1412  skip_spaces();
1413  std::string detail = read_rest_of_line();
1414  deprecated_message(get_filename(parent_.location_), level, version, detail);
1415  } else {
1416  comment = token.type != token_desc::MACRO_SPACE;
1417  }
1418 
1419  skip_eol();
1420  if(comment) {
1421  put('\n');
1422  }
1423  } else if(token.type == token_desc::MACRO_SPACE || token.type == token_desc::MACRO_CHUNK) {
1424  if(c == '(') {
1425  // If a macro argument was started, it is implicitly ended.
1426  token.type = token_desc::MACRO_SPACE;
1428  } else if(utils::portable_isspace(c)) {
1429  // If a macro argument was started, it is implicitly ended.
1430  token.type = token_desc::MACRO_SPACE;
1431  } else if(c == '}') {
1432  --slowpath_;
1433  if(skipping_) {
1434  pop_token();
1435  return true;
1436  }
1437 
1438  // FIXME: is this obsolete?
1439  // if (token.type == token_desc::MACRO_SPACE) {
1440  // if (!strings_.back().empty()) {
1441  // std::ostringstream error;
1442  // std::ostringstream location;
1443  // error << "Can't parse new macro parameter with a macro call scope open";
1444  // location<<linenum_<<' '<<parent_.location_;
1445  // parent_.error(error.str(), location.str());
1446  // }
1447  // strings_.pop_back();
1448  //}
1449 
1450  if(strings_.size() <= static_cast<std::size_t>(token.stack_pos)) {
1451  parent_.error("No macro or file substitution target specified", linenum_);
1452  }
1453 
1454  std::string symbol = strings_[token.stack_pos];
1455  std::string::size_type pos;
1456  while((pos = symbol.find(OUTPUT_SEPARATOR)) != std::string::npos) {
1457  std::string::iterator b = symbol.begin(); // invalidated at each iteration
1458  symbol.erase(b + pos, b + symbol.find('\n', pos + 1) + 1);
1459  }
1460 
1461  std::map<std::string, std::string>::const_iterator arg;
1462  preproc_map::const_iterator macro;
1463 
1464  // If this is a known pre-processing symbol, then we insert it,
1465  // otherwise we assume it's a file name to load.
1466  if(symbol == current_file_str && strings_.size() - token.stack_pos == 1) {
1467  pop_token();
1469  } else if(symbol == current_dir_str && strings_.size() - token.stack_pos == 1) {
1470  pop_token();
1472  } else if(symbol == left_curly_str && strings_.size() - token.stack_pos == 1) {
1473  pop_token();
1474  put("{");
1475  } else if(symbol == right_curly_str && strings_.size() - token.stack_pos == 1) {
1476  pop_token();
1477  put("}");
1478  } else if(local_defines_ && (arg = local_defines_->find(symbol)) != local_defines_->end()) {
1479  if(strings_.size() - token.stack_pos != 1) {
1480  std::ostringstream error;
1481  error << "Macro argument '" << symbol << "' does not expect any arguments";
1482  parent_.error(error.str(), linenum_);
1483  }
1484 
1485  std::ostringstream v;
1486  v << arg->second << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1487  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1488 
1489  pop_token();
1490  put(v.str());
1491  } else if(parent_.depth() < 100 && (macro = parent_.defines_->find(symbol)) != parent_.defines_->end()) {
1492  const preproc_define& val = macro->second;
1493  std::size_t nb_arg = strings_.size() - token.stack_pos - 1;
1494  std::size_t optional_arg_num = 0;
1495 
1496  std::unique_ptr<std::map<std::string, std::string>> defines{new std::map<std::string, std::string>};
1497  const std::string& dir = filesystem::directory_name(val.location.substr(0, val.location.find(' ')));
1498 
1499  if(val.is_deprecated()) {
1501  }
1502 
1503  for(std::size_t i = 0; i < nb_arg; ++i) {
1504  if(i < val.arguments.size()) {
1505  // Normal mandatory arguments
1506 
1507  (*defines)[val.arguments[i]] = strings_[token.stack_pos + i + 1];
1508  } else {
1509  // These should be optional argument overrides
1510 
1511  std::string str = strings_[token.stack_pos + i + 1];
1512  std::size_t equals_pos = str.find_first_of("=");
1513 
1514  if(equals_pos != std::string::npos) {
1515  std::size_t argname_pos = str.substr(0, equals_pos).find_last_of(" \n") + 1;
1516 
1517  std::string argname = str.substr(argname_pos, equals_pos - argname_pos);
1518 
1519  if(val.optional_arguments.find(argname) != val.optional_arguments.end()) {
1520  (*defines)[argname] = str.substr(equals_pos + 1);
1521 
1522  optional_arg_num++;
1523 
1524  DBG_PREPROC << "Found override for " << argname << " in call to macro " << symbol
1525  << "\n";
1526  } else {
1527  std::ostringstream warning;
1528  warning << "Unrecognized optional argument passed to macro '" << symbol << "': '"
1529  << argname << "'";
1530  parent_.warning(warning.str(), linenum_);
1531 
1532  optional_arg_num++; // To prevent the argument number check from blowing up
1533  }
1534  }
1535  }
1536  }
1537 
1538  // If the macro definition has any optional arguments, insert their defaults
1539  if(val.optional_arguments.size() > 0) {
1540  for(const auto& argument : val.optional_arguments) {
1541  if(defines->find(argument.first) == defines->end()) {
1542  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1543 
1545  std::istream in(buf.get());
1546 
1547  filesystem::scoped_istream buffer{new std::istringstream(argument.second)};
1548 
1549  std::unique_ptr<std::map<std::string, std::string>> temp_defines{new std::map<std::string, std::string>};
1550  temp_defines->insert(defines->begin(), defines->end());
1551 
1553  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(temp_defines), false);
1554 
1555  std::ostringstream res;
1556  res << in.rdbuf();
1557 
1558  DBG_PREPROC << "Setting default for optional argument " << argument.first << " in macro "
1559  << symbol << "\n";
1560 
1561  (*defines)[argument.first] = res.str();
1562  }
1563  }
1564  }
1565 
1566  if(nb_arg - optional_arg_num != val.arguments.size()) {
1567  const std::vector<std::string>& locations = utils::quoted_split(val.location, ' ');
1568  const std::string filename = locations.empty() ? "<command-line>" : get_filename(locations[0]);
1569  std::ostringstream error;
1570  error << "Preprocessor symbol '" << symbol << "' defined at " << filename << ":"
1571  << val.linenum << " expects " << val.arguments.size() << " arguments, but has "
1572  << nb_arg - optional_arg_num << " arguments";
1573  parent_.error(error.str(), linenum_);
1574  }
1575 
1576  filesystem::scoped_istream buffer{new std::istringstream(val.value)};
1577 
1578  pop_token();
1579 
1580  if(!slowpath_) {
1581  DBG_PREPROC << "substituting macro " << symbol << '\n';
1582 
1584  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1585  } else {
1586  DBG_PREPROC << "substituting (slow) macro " << symbol << '\n';
1587 
1588  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1589 
1590  // Make the nested preprocessor_data responsible for
1591  // restoring our current textdomain if needed.
1593 
1594  std::ostringstream res;
1595  {
1596  std::istream in(buf.get());
1598  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1599 
1600  res << in.rdbuf();
1601  }
1602 
1603  put(res.str());
1604  }
1605  } else if(parent_.depth() < 40) {
1606  LOG_PREPROC << "Macro definition not found for " << symbol << " , attempting to open as file.\n";
1607  pop_token();
1608 
1609  std::string nfname = filesystem::get_wml_location(symbol, directory_);
1610  if(!nfname.empty()) {
1611  if(!slowpath_)
1612  // nfname.size() - symbol.size() gives you an index into nfname
1613  // This does not necessarily match the symbol though, as it can start with ~ or ./
1614  parent_.add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1615  else {
1616  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1617 
1618  std::ostringstream res;
1619  {
1620  std::istream in(buf.get());
1621  buf->add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1622 
1623  res << in.rdbuf();
1624  }
1625 
1626  put(res.str());
1627  }
1628  } else {
1629  std::ostringstream error;
1630  error << "Macro/file '" << symbol << "' is missing";
1631  parent_.error(error.str(), linenum_);
1632  }
1633  } else {
1634  parent_.error("Too many nested preprocessing inclusions", linenum_);
1635  }
1636  } else if(!skipping_) {
1637  if(token.type == token_desc::MACRO_SPACE) {
1638  std::ostringstream s;
1639  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1640  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1641 
1642  strings_.push_back(s.str());
1643  token.type = token_desc::MACRO_CHUNK;
1644  }
1645  put(c);
1646  }
1647  } else {
1648  put(c);
1649  }
1650 
1651  return true;
1652 }
1653 
1654 
1655 // ==================================================================================
1656 // PREPROCESSOR SCOPE HELPER
1657 // ==================================================================================
1658 
1659 struct preprocessor_scope_helper : std::basic_istream<char>
1660 {
1661  preprocessor_scope_helper(const std::string& fname, preproc_map* defines)
1662  : std::basic_istream<char>(nullptr)
1663  , buf_(nullptr)
1664  , local_defines_(nullptr)
1665  {
1666  //
1667  // If no defines were provided, we create a new local preproc_map and assign
1668  // it to defines temporarily. In this case, the map will be deleted once this
1669  // object is destroyed and defines will still be subsequently null.
1670  //
1671  if(!defines) {
1672  local_defines_.reset(new preproc_map);
1673  defines = local_defines_.get();
1674  }
1675 
1676  buf_.reset(new preprocessor_streambuf(defines));
1677 
1678  // Begin processing.
1679  buf_->add_preprocessor<preprocessor_file>(fname);
1680 
1681  //
1682  // TODO: not sure if this call is needed. Previously, this call was passed a
1683  // preprocessor_streambuf pointer and the std::basic_istream constructor was
1684  // called with its contents. However, at that point the preprocessing should
1685  // already have completed, meaning this call might be redundant. Not sure.
1686  //
1687  // - vultraz, 2017-08-31
1688  //
1689  init(buf_.get());
1690  }
1691 
1693  {
1694  clear(std::ios_base::goodbit);
1695  exceptions(std::ios_base::goodbit);
1696  rdbuf(nullptr);
1697  }
1698 
1699  std::unique_ptr<preprocessor_streambuf> buf_;
1700  std::unique_ptr<preproc_map> local_defines_;
1701 };
1702 
1703 
1704 // ==================================================================================
1705 // FREE-STANDING FUNCTIONS
1706 // ==================================================================================
1707 
1708 filesystem::scoped_istream preprocess_file(const std::string& fname, preproc_map* defines)
1709 {
1710  log_scope("preprocessing file " + fname + " ...");
1711 
1712  // NOTE: the preprocessor_scope_helper does *not* take ownership of defines.
1713  return filesystem::scoped_istream(new preprocessor_scope_helper(fname, defines));
1714 }
1715 
1716 void preprocess_resource(const std::string& res_name,
1717  preproc_map* defines_map,
1718  bool write_cfg,
1719  bool write_plain_cfg,
1720  const std::string& parent_directory)
1721 {
1722  if(filesystem::is_directory(res_name)) {
1723  std::vector<std::string> dirs, files;
1724 
1727 
1728  // Subdirectories
1729  for(const std::string& dir : dirs) {
1730  LOG_PREPROC << "processing sub-dir: " << dir << '\n';
1731  preprocess_resource(dir, defines_map, write_cfg, write_plain_cfg, parent_directory);
1732  }
1733 
1734  // Files in current directory
1735  for(const std::string& file : files) {
1736  preprocess_resource(file, defines_map, write_cfg, write_plain_cfg, parent_directory);
1737  }
1738 
1739  return;
1740  }
1741 
1742  // process only config files.
1743  if(!filesystem::ends_with(res_name, ".cfg")) {
1744  return;
1745  }
1746 
1747  LOG_PREPROC << "processing resource: " << res_name << '\n';
1748 
1749  // disable filename encoding to get clear #line in cfg.plain
1750  encode_filename = false;
1751 
1752  filesystem::scoped_istream stream = preprocess_file(res_name, defines_map);
1753 
1754  std::stringstream ss;
1755 
1756  // Set the failbit so if we get any preprocessor exceptions (e.g.:preproc_config::error)
1757  // they will be propagated in the main program, instead of just setting the
1758  // failbit on the stream. This was necessary in order for the MSVC and GCC
1759  // binaries to behave the same way.
1760  ss.exceptions(std::ios_base::failbit);
1761 
1762  ss << (*stream).rdbuf();
1763 
1764  LOG_PREPROC << "processing finished\n";
1765 
1766  if(write_cfg || write_plain_cfg) {
1767  config cfg;
1768  std::string streamContent = ss.str();
1769 
1770  read(cfg, streamContent);
1771 
1772  const std::string preproc_res_name = parent_directory + "/" + filesystem::base_name(res_name);
1773 
1774  // Write the processed cfg file
1775  if(write_cfg) {
1776  LOG_PREPROC << "writing cfg file: " << preproc_res_name << '\n';
1777 
1779  filesystem::scoped_ostream outStream(filesystem::ostream_file(preproc_res_name));
1780 
1781  write(*outStream, cfg);
1782  }
1783 
1784  // Write the plain cfg file
1785  if(write_plain_cfg) {
1786  LOG_PREPROC << "writing plain cfg file: " << (preproc_res_name + ".plain") << '\n';
1787 
1789  filesystem::write_file(preproc_res_name + ".plain", streamContent);
1790  }
1791  }
1792 }
std::string lineno_string(const std::string &lineno)
VERSION_COMP_OP
static const char OUTPUT_SEPARATOR
std::vector< std::string > arguments
std::optional< DEP_LEVEL > deprecation_level
preprocessor * current() const
preprocessor_file(preprocessor_streambuf &t, const std::string &name, std::size_t symbol_index=-1)
Constructor.
void write(config_writer &, const std::string &) const
std::vector< std::string > files_
virtual bool get_chunk() override
Inserts and processes the next file in the list of included files.
preprocessor_scope_helper(const std::string &fname, preproc_map *defines)
Specialized preprocessor for handling a file or a set of files.
void read(const config &)
Interfaces for manipulating version numbers of engine, add-ons, etc.
static const std::string current_file_str
static int read_line(lua_State *L, FILE *f, int chop)
Definition: liolib.cpp:513
std::vector< std::string >::const_iterator end_
VERSION_COMP_OP parse_version_op(const std::string &op_str)
std::string read_word()
preproc_map default_defines_
static l_noret error(LoadState *S, const char *why)
Definition: lundump.cpp:40
buffered_istream in_
Input stream.
static std::map< std::string, int > file_number_map
bool ends_with(const std::string &str, const std::string &suffix)
#define ERR_PREPROC
child_itors child_range(config_key_type key)
Definition: config.cpp:356
virtual MODE parse_mode()
Returns the appropriate parsing mode for this preprocessor.
int peek()
Gets a character from the buffer.
Specialized preprocessor for handling any kind of input stream.
std::string out_buffer_
Buffer read by the STL stream.
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
void clear(const std::string &key)
Definition: general.cpp:203
Helper class for buffering a std::istream.
virtual bool get_chunk() override
Preprocesses and sends some text to the parent_ buffer.
void preprocess_resource(const std::string &res_name, preproc_map *defines_map, bool write_cfg, bool write_plain_cfg, const std::string &parent_directory)
STL namespace.
void read_argument(const config &)
const std::vector< std::string > items
#define d
std::unique_ptr< preprocessor_streambuf > buf_
bool do_version_check(const version_info &a, VERSION_COMP_OP op, const version_info &b)
Definitions for the interface to Wesnoth Markup Language (WML).
std::vector< std::string >::const_iterator pos_
filesystem::scoped_ostream ostream_file(const std::string &fname, std::ios_base::openmode mode, bool create_directory)
std::string location
Description of a preprocessing chunk.
#define WRN_PREPROC
#define b
void write_argument(config_writer &, const std::string &) const
unsigned in
If equal to search_counter, the node is off the list.
void write(std::ostream &out, const configr_of &cfg, unsigned int level)
Definition: parser.cpp:763
int slowpath_
Set to true whenever input tokens cannot be directly sent to the target buffer.
bool is_define_
True iff we are currently parsing a macros content, otherwise false.
void write_key_val(const std::string &key, const T &value)
This template function will work with any type that can be assigned to an attribute_value.
int get()
Gets and consumes a character from the buffer.
void conditional_skip(bool skip)
std::string deprecated_message(const std::string &elem_name, DEP_LEVEL level, const version_info &version, const std::string &detail)
Definition: deprecation.cpp:29
bool quoted_
Set to true if one preprocessor for this target started to read a string.
void close_child(const std::string &key)
void add_preprocessor(A &&... args)
static lg::log_domain log_preprocessor("preprocessor")
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:626
Class for writing a config out to a file in pieces.
version_info deprecation_version
void restore_old_preprocessor()
Restores the old preprocessing context.
std::string read_line()
void write_file(const std::string &fname, const std::string &data)
Throws io_exception if an error occurs.
int skipping_
Non-zero when the preprocessor has to skip some input text.
void push_token(token_desc::TOKEN_TYPE)
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, name_mode mode, filter_mode filter, reorder_mode reorder, file_tree_checksum *checksum)
Populates &#39;files&#39; with all the files and &#39;dirs&#39; with all the directories in dir.
Definition: filesystem.cpp:349
void open_child(const std::string &key)
std::deque< std::unique_ptr< preprocessor > > preprocessor_queue_
Input preprocessor queue.
virtual ~preprocessor()
bool is_deprecated() const
bool is_directory(const std::string &fname)
Returns true if the given file is a directory.
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:37
virtual void init() override
Allows specifying any actions that need to be called after the constructor completes.
std::string get_short_wml_path(const std::string &filename)
Returns a short path to filename, skipping the (user) data directory.
std::string deprecation_message
#define DBG_PREPROC
std::string value
#define LOG_PREPROC
static int writer(lua_State *L, const void *b, size_t size, void *ud)
Definition: lstrlib.cpp:221
std::string escape(const std::string &str, const char *special_chars)
Prepends a configurable set of characters with a backslash.
#define PACKAGE
Definition: wesconfig.h:23
std::string directory_
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:38
virtual preprocessor::MODE parse_mode() override
Returns the appropriate parsing mode for this preprocessor.
static std::string get_file_code(const std::string &filename)
Some defines: VERSION, PACKAGE, MIN_SAVEGAME_VERSION.
virtual void init()
Allows specifying any actions that need to be called after the constructor completes.
bool operator==(const preproc_define &) const
token_desc(TOKEN_TYPE type, const int stack_pos, const int linenum)
std::size_t i
Definition: function.cpp:940
logger & err()
Definition: log.cpp:76
std::string get_wml_location(const std::string &filename, const std::string &current_dir)
Returns a complete path to the actual WML file or directory or an empty string if the file isn&#39;t pres...
std::string read_rest_of_line()
std::unique_ptr< preproc_map > local_defines_
bool operator!=(preprocessor_data::token_desc::TOKEN_TYPE rhs, char lhs)
mock_party p
static std::string get_location(const std::string &loc)
static map_location::DIRECTION s
filesystem::scoped_istream in_scope_
Manages the lifetime of the std::istream pointer we own.
std::vector< std::string > quoted_split(const std::string &val, char c, int flags, char quote)
This function is identical to split(), except it does not split when it otherwise would if the previo...
friend class preprocessor_streambuf
#define log_scope(description)
Definition: log.hpp:206
Helper class for buffering a std::istream.
std::unique_ptr< std::map< std::string, std::string > > local_defines_
Mapping of macro arguments to their content.
Declarations for File-IO.
void warning(const std::string &, int)
const version_info wesnoth_version(VERSION)
Base class for preprocessing an input.
std::vector< std::string > strings_
Buffer for delayed input processing.
Represents version numbers.
std::ostream & operator<<(std::ostream &stream, const preproc_define &def)
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
preprocessor_streambuf(const preprocessor_streambuf &t)
static std::string get_filename(const std::string &file_code)
double t
Definition: astarsearch.cpp:64
std::vector< std::string > split(const config_attribute_value &val)
DEP_LEVEL
See https://wiki.wesnoth.org/CompatibilityStandards for more info.
Definition: deprecation.hpp:19
std::vector< token_desc > tokens_
Stack of nested preprocessing chunks.
static const std::string current_dir_str
preprocessor_streambuf & parent_
Standard logging facilities (interface).
std::string textdomain
std::string str() const
Serializes the version number into string form.
virtual int underflow() override
Inherited from basic_streambuf.
virtual bool get_chunk()=0
Preprocesses and sends some text to the parent_ buffer.
int stack_pos
Starting position in strings_ of the delayed text for this chunk.
preprocessor_data(preprocessor_streambuf &, filesystem::scoped_istream, const std::string &history, const std::string &name, int line, const std::string &dir, const std::string &domain, std::unique_ptr< std::map< std::string, std::string >> defines, bool is_define=false)
void error(const std::string &, int)
Target for sending preprocessed output.
std::map< std::string, struct preproc_define > preproc_map
#define e
static const std::string right_curly_str
bool portable_isspace(const char c)
preprocessor_streambuf(preproc_map *def)
bool operator==(preprocessor_data::token_desc::TOKEN_TYPE, char)
std::string get_current_file()
Decodes the filenames placed in a location.
std::string old_location_
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:59
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
Function to use the WML preprocessor on a file.
static preproc_map::value_type read_pair(const config &)
mock_char c
const std::string & name_
std::string old_textdomain_
static map_location::DIRECTION n
bool operator<(const preproc_define &) const
static bool encode_filename
std::string::const_iterator iterator
Definition: tokenizer.hpp:24
std::stringstream buffer_
Buffer filled by the current preprocessor.
preprocessor(preprocessor_streambuf &t)
Sets up a new preprocessor for stream buffer t.
std::string directory_name(const std::string &file)
Returns the directory name of a file, with filename stripped.
bool eof() const
Is the end of input reached?
static const std::string left_curly_str
static bool create_directory_if_missing_recursive(const bfs::path &dirpath)
Definition: filesystem.cpp:321
static std::string preprocessor_error_detail_prefix
std::map< std::string, std::string > optional_arguments