The Battle for Wesnoth  1.19.15+dev
preprocessor.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2005 - 2025
3  by Guillaume Melquiond <guillaume.melquiond@gmail.com>
4  Copyright (C) 2003 by David White <dave@whitevine.net>
5  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
6 
7  This program is free software; you can redistribute it and/or modify
8  it under the terms of the GNU General Public License as published by
9  the Free Software Foundation; either version 2 of the License, or
10  (at your option) any later version.
11  This program is distributed in the hope that it will be useful,
12  but WITHOUT ANY WARRANTY.
13 
14  See the COPYING file for more details.
15 */
16 
17 /**
18  * @file
19  * WML preprocessor.
20  */
21 
23 
24 #include "buffered_istream.hpp"
25 #include "config.hpp"
26 #include "log.hpp"
28 #include "serialization/parser.hpp"
30 #include "game_version.hpp"
31 #include "wesconfig.h"
32 #include "deprecation.hpp"
33 
34 #include <stdexcept>
35 #include <deque>
36 
37 static lg::log_domain log_preprocessor("preprocessor");
38 #define ERR_PREPROC LOG_STREAM(err, log_preprocessor)
39 #define WRN_PREPROC LOG_STREAM(warn, log_preprocessor)
40 #define LOG_PREPROC LOG_STREAM(info, log_preprocessor)
41 #define DBG_PREPROC LOG_STREAM(debug, log_preprocessor)
42 
43 static const std::string current_file_str = "CURRENT_FILE";
44 static const std::string current_dir_str = "CURRENT_DIRECTORY";
45 static const std::string left_curly_str = "LEFT_BRACE";
46 static const std::string right_curly_str = "RIGHT_BRACE";
47 
48 // map associating each filename encountered to a number
49 static std::map<std::string, int> file_number_map;
50 
51 static bool encode_filename = true;
52 
53 static std::string preprocessor_error_detail_prefix = "\n ";
54 
55 // get filename associated to this code
56 static std::string get_filename(const std::string& file_code)
57 {
58  if(!encode_filename) {
59  return file_code;
60  }
61 
62  std::stringstream s;
63  s << file_code;
64  int n = 0;
65  s >> std::hex >> n;
66 
67  for(const auto& p : file_number_map) {
68  if(p.second == n) {
69  return p.first;
70  }
71  }
72 
73  return "<unknown>";
74 }
75 
76 // Get code associated to this filename
77 static std::string get_file_code(const std::string& filename)
78 {
79  if(!encode_filename) {
80  return filename;
81  }
82 
83  // Current number of encountered filenames
84  static int current_file_number = 0;
85 
86  int& fnum = file_number_map[utils::escape(filename, " \\")];
87  if(fnum == 0) {
88  fnum = ++current_file_number;
89  }
90 
91  std::ostringstream shex;
92  shex << std::hex << fnum;
93 
94  return shex.str();
95 }
96 
97 // decode the filenames placed in a location
98 static std::string get_location(const std::string& loc)
99 {
100  std::string res;
101  std::vector<std::string> pos = utils::quoted_split(loc, ' ');
102 
103  if(pos.empty()) {
104  return res;
105  }
106 
107  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
108  while(true) {
109  res += get_filename(*(i++));
110 
111  if(i == end) {
112  break;
113  }
114 
115  res += ' ';
116  res += *(i++);
117 
118  if(i == end) {
119  break;
120  }
121 
122  res += ' ';
123  }
124 
125  return res;
126 }
127 
128 
129 // ==================================================================================
130 // PREPROC_DEFINE IMPLEMENTATION
131 // ==================================================================================
132 
134 {
135  return value == v.value && arguments == v.arguments;
136 }
137 
139 {
140  if(location < v.location) {
141  return true;
142  }
143 
144  if(v.location < location) {
145  return false;
146  }
147 
148  return linenum < v.linenum;
149 }
150 
151 void preproc_define::write_argument(config_writer& writer, const std::string& arg) const
152 {
153  const std::string key = "argument";
154 
155  writer.open_child(key);
156 
157  writer.write_key_val("name", arg);
158  writer.close_child(key);
159 }
160 
161 void preproc_define::write_argument(config_writer& writer, const std::string& arg, const std::string& default_value) const
162 {
163  const std::string key = "argument";
164 
165  writer.open_child(key);
166 
167  writer.write_key_val("name", arg);
168  writer.write_key_val("default", default_value);
169  writer.close_child(key);
170 }
171 
172 void preproc_define::write(config_writer& writer, const std::string& name) const
173 {
174  const std::string key = "preproc_define";
175  writer.open_child(key);
176 
177  writer.write_key_val("name", name);
178  writer.write_key_val("value", value);
179  writer.write_key_val("textdomain", textdomain);
180  writer.write_key_val("linenum", std::to_string(linenum));
181  writer.write_key_val("location", get_location(location));
182 
183  if(is_deprecated()) {
184  writer.open_child("deprecated");
185  writer.write_key_val("level", int(*deprecation_level));
186  writer.write_key_val("version", deprecation_version.str());
187  writer.write_key_val("message", deprecation_message);
188  writer.close_child("deprecated");
189  }
190 
191  for(const std::string& arg : arguments) {
192  write_argument(writer, arg);
193  }
194 
195  for(const auto& [key, default_value] : optional_arguments) {
196  write_argument(writer, key, default_value);
197  }
198 
199  writer.close_child(key);
200 }
201 
203 {
204  if(cfg.has_attribute("default")) {
205  optional_arguments.emplace(cfg["name"], cfg["default"]);
206  } else {
207  arguments.push_back(cfg["name"]);
208  }
209 }
210 
212 {
213  value = cfg["value"].str();
214  textdomain = cfg["textdomain"].str();
215  linenum = cfg["linenum"].to_int();
216  location = cfg["location"].str();
217 
218  if(auto deprecated = cfg.optional_child("deprecated")) {
219  deprecation_level = DEP_LEVEL(deprecated.value()["level"].to_int());
220  deprecation_version = deprecated.value()["version"].str();
221  deprecation_message = deprecated.value()["message"].str();
222  }
223 
224  for(const config& arg : cfg.child_range("argument")) {
225  read_argument(arg);
226  }
227 }
228 
230 {
231  map.try_emplace(cfg["name"], cfg);
232 }
233 
234 std::ostream& operator<<(std::ostream& stream, const preproc_define& def)
235 {
236  return stream << "value: " << def.value << " arguments: " << def.location;
237 }
238 
239 std::ostream& operator<<(std::ostream& stream, const preproc_map::value_type& def)
240 {
241  return stream << def.second;
242 }
243 
244 // ==================================================================================
245 // PREPROCESSOR BASE
246 // ==================================================================================
247 
249 
250 /**
251  * Base class for preprocessing an input.
252  */
254 {
256 
257 protected:
258  /**
259  * Sets up a new preprocessor for stream buffer \a t.
260  * Saves the current preprocessing context of #parent_. It will be automatically restored on destruction.
261  *
262  * It relies on preprocessor_streambuf so it's implemented after that class is declared.
263  */
265 
267 
268 public:
269  virtual ~preprocessor()
270  {
271  }
272 
273  /** Allows specifying any actions that need to be called after the constructor completes. */
274  virtual void init()
275  {
276  }
277 
278  /**
279  * Preprocesses and sends some text to the #parent_ buffer.
280  * @return false when the input has no data left.
281  */
282  virtual bool get_chunk() = 0;
283 
285 
286  /** Returns the appropriate parsing mode for this preprocessor. */
287  virtual MODE parse_mode()
288  {
289  return NO_PARSING;
290  }
291 
292 private:
293  std::string old_textdomain_;
294  std::string old_location_;
295 
297 };
298 
299 
300 // ==================================================================================
301 // PREPROCESSOR BUFFER
302 // ==================================================================================
303 
304 /**
305  * Target for sending preprocessed output.
306  * Objects of this class can be plugged into an STL stream.
307  */
308 class preprocessor_streambuf : public std::streambuf
309 {
310 public:
312  : std::streambuf()
313  , out_buffer_("")
314  , buffer_()
316  , defines_(def)
318  , location_("")
319  , linenum_(0)
320  , quoted_(false)
321  {
322  }
323 
324  /** Decodes the filenames placed in a location. */
325  std::string get_current_file();
326 
327  void error(const std::string&, int);
328  void warning(const std::string&, int);
329 
330  template<typename T, typename... A>
331  void add_preprocessor(A&&... args)
332  {
333  preprocessor_queue_.emplace_back(new T(*this, std::forward<A>(args)...));
334  preprocessor_queue_.back()->init();
335  }
336 
338  {
339  preprocessor_queue_.pop_back();
340  }
341 
342  int depth() const
343  {
344  return preprocessor_queue_.size();
345  }
346 
348  {
349  return preprocessor_queue_.empty() ? nullptr : preprocessor_queue_.back().get();
350  }
351 
352 private:
354  : std::streambuf()
355  , out_buffer_("")
356  , buffer_()
358  , defines_(t.defines_)
360  , location_("")
361  , linenum_(0)
362  , quoted_(t.quoted_)
363  {
364  }
365 
366  /** Inherited from basic_streambuf. */
367  virtual int underflow() override;
368 
370 
371  /** Buffer read by the STL stream. */
372  std::string out_buffer_;
373 
374  /** Buffer filled by the _current_ preprocessor. */
375  std::stringstream buffer_;
376 
377  /** Input preprocessor queue. */
378  std::deque<std::unique_ptr<preprocessor>> preprocessor_queue_;
379 
381 
382  std::string textdomain_;
383  std::string location_;
384 
385  int linenum_;
386 
387  /**
388  * Set to true if one preprocessor for this target started to read a string.
389  * Deeper-nested preprocessors are then forbidden to.
390  */
391  bool quoted_;
392 
393  friend class preprocessor;
394  friend class preprocessor_file;
395  friend class preprocessor_data;
396  friend class preprocessor_istream;
397 };
398 
399 /** Preprocessor constructor. */
401  : parent_(t)
402  , old_textdomain_(t.textdomain_)
403  , old_location_(t.location_)
404  , old_linenum_(t.linenum_)
405 {
406 }
407 
408 /**
409  * Called by an STL stream whenever it has reached the end of #out_buffer_.
410  * Fills #buffer_ by calling the _current_ preprocessor, then copies its
411  * content into #out_buffer_.
412  * @return the first character of #out_buffer_ if any, EOF otherwise.
413  */
415 {
416  unsigned sz = 0;
417  if(char* gp = gptr()) {
418  if(gp < egptr()) {
419  // Sanity check: the internal buffer has not been totally consumed,
420  // should we force the caller to use what remains first?
421  return *gp;
422  }
423 
424  // The buffer has been completely read; fill it again.
425  // Keep part of the previous buffer, to ensure putback capabilities.
426  sz = out_buffer_.size();
427  buffer_.str(std::string());
428 
429  if(sz > 3) {
430  buffer_ << out_buffer_.substr(sz - 3);
431  sz = 3;
432  } else {
433  buffer_ << out_buffer_;
434  }
435  } else {
436  // The internal get-data pointer is null
437  }
438 
439  const int desired_fill_amount = 2000;
440 
441  while(current() && buffer_.rdbuf()->in_avail() < desired_fill_amount) {
442  // Process files and data chunks until the desired buffer size is reached
443  if(!current()->get_chunk()) {
444  // Drop the current preprocessor item from the queue.
446  }
447  }
448 
449  // Update the internal state and data pointers
450  out_buffer_ = buffer_.str();
451  if(out_buffer_.empty()) {
452  return EOF;
453  }
454 
455  char* begin = &*out_buffer_.begin();
456  unsigned bs = out_buffer_.size();
457 
458  setg(begin, begin + sz, begin + bs);
459 
460  if(sz >= bs) {
461  return EOF;
462  }
463 
464  return static_cast<unsigned char>(*(begin + sz));
465 }
466 
467 /**
468 * Restores the old preprocessing context.
469 * Appends location and domain directives to the buffer, so that the parser
470 * notices these changes.
471 */
473 {
474  preprocessor* current = this->current();
475 
476  if(!current->old_location_.empty()) {
478  }
479 
481  buffer_ << INLINED_PREPROCESS_DIRECTIVE_CHAR << "textdomain " << current->old_textdomain_ << '\n';
482  }
483 
487 
488  // Drop the preprocessor from the queue.
490 }
491 
493 {
494  unsigned nested_level = 0;
495 
496  preprocessor* pre = nullptr;
497 
498  // Iterate backwards over queue to get the last non-macro preprocessor.
499  for(auto p = preprocessor_queue_.rbegin(); p != preprocessor_queue_.rend(); ++p) {
500  pre = p->get();
501 
502  if(!pre || pre->parse_mode() == preprocessor::PARSES_FILE) {
503  break;
504  }
505 
506  if(pre->parse_mode() == preprocessor::PARSES_MACRO) {
507  ++nested_level;
508  }
509  }
510 
511  std::string res;
512  std::vector<std::string> pos = utils::quoted_split(location_, ' ');
513 
514  if(pos.size() <= 2 * nested_level) {
515  return res;
516  }
517 
518  return get_filename(pos[2 * nested_level]);
519 }
520 
521 std::string lineno_string(const std::string& lineno)
522 {
523  std::vector<std::string> pos = utils::quoted_split(lineno, ' ');
524  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
525  std::string included_from = preprocessor_error_detail_prefix + "included from ";
526  std::string res;
527 
528  while(i != end) {
529  const std::string& line = *(i++);
530 
531  if(!res.empty()) {
532  res += included_from;
533  }
534 
535  if(i != end) {
536  res += get_filename(*(i++));
537  } else {
538  res += "<unknown>";
539  }
540 
541  res += ':' + line;
542  }
543 
544  if(res.empty()) {
545  res = "???";
546  }
547 
548  return res;
549 }
550 
551 void preprocessor_streambuf::error(const std::string& error_type, int l)
552 {
553  std::string position, error;
554  std::ostringstream pos;
555 
556  pos << l << ' ' << location_;
557  position = lineno_string(pos.str());
558 
559  error = error_type + '\n';
560  error += "at " + position;
561 
562  ERR_PREPROC << error;
563 
565 }
566 
567 void preprocessor_streambuf::warning(const std::string& warning_type, int l)
568 {
569  std::string position, warning;
570  std::ostringstream pos;
571 
572  pos << l << ' ' << location_;
573  position = lineno_string(pos.str());
574 
575  warning = warning_type + '\n';
576  warning += "at " + position;
577 
578  WRN_PREPROC << warning;
579 }
580 
581 
582 // ==================================================================================
583 // PREPROCESSOR FILE
584 // ==================================================================================
585 
586 /**
587  * Specialized preprocessor for handling a file or a set of files.
588  * A preprocessor_file object is created when a preprocessor encounters an
589  * inclusion directive that resolves to a file or directory, e.g. '{themes/}'.
590  */
592 {
593 public:
594  /** Constructor. It relies on preprocessor_data so it's implemented after that class is declared. */
595  preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index = -1);
596 
597  virtual void init() override;
598 
599  /**
600  * Inserts and processes the next file in the list of included files.
601  * @return false if there is no next file.
602  */
603  virtual bool get_chunk() override
604  {
605  while(pos_ != end_) {
606  const std::string& name = *(pos_++);
607  unsigned sz = name.size();
608 
609  // Use reverse iterator to optimize testing
610  if(sz < 5 || !std::equal(name.rbegin(), name.rbegin() + 4, "gfc.")) {
611  continue;
612  }
613 
615  return true;
616  }
617 
618  return false;
619  }
620 
621 private:
622  std::vector<std::string> files_;
623  std::vector<std::string>::const_iterator pos_, end_;
624 
625  const std::string& name_;
626 
628 };
629 
630 
631 // ==================================================================================
632 // PREPROCESSOR DATA
633 // ==================================================================================
634 
635 /**
636  * Specialized preprocessor for handling any kind of input stream.
637  * This is the core of the preprocessor.
638  */
640 {
641  /** Description of a preprocessing chunk. */
642  struct token_desc
643  {
644  enum class token_type {
645  start, // Toplevel
646  process_if, // Processing the "if" branch of a ifdef/ifndef (the "else" branch will be skipped)
647  process_else, // Processing the "else" branch of a ifdef/ifndef
648  skip_if, // Skipping the "if" branch of a ifdef/ifndef (the "else" branch, if any, will be processed)
649  skip_else, // Skipping the "else" branch of a ifdef/ifndef
650  string, // Processing a string
651  verbatim, // Processing a verbatim string
652  macro_space, // Processing between chunks of a macro call (skip spaces)
653  macro_chunk, // Processing inside a chunk of a macro call (stop on space or '(')
654  macro_parens // Processing a parenthesized macro argument
655  };
656 
657  token_desc(token_type type, const int stack_pos, const int linenum)
658  : type(type)
660  , linenum(linenum)
661  {
662  }
663 
665 
666  /** Starting position in #strings_ of the delayed text for this chunk. */
668  int linenum;
669  };
670 
671  /**
672  * Manages the lifetime of the @c std::istream pointer we own.
673  *
674  * Since @ref in_ uses the stream as well this object must be created
675  * before @ref in_ and destroyed after @ref in_ is destroyed.
676  */
678 
679  /** Input stream. */
681 
682  std::string directory_;
683 
684  /** Buffer for delayed input processing. */
685  std::vector<std::string> strings_;
686 
687  /** Mapping of macro arguments to their content. */
688  std::unique_ptr<std::map<std::string, std::string>> local_defines_;
689 
690  /** Stack of nested preprocessing chunks. */
691  std::vector<token_desc> tokens_;
692 
693  /**
694  * Set to true whenever input tokens cannot be directly sent to the target
695  * buffer. For instance, this happens with macro arguments. In that case,
696  * the output is redirected toward #strings_ until it can be processed.
697  */
699 
700  /**
701  * Non-zero when the preprocessor has to skip some input text.
702  * Increased whenever entering a conditional branch that is not useful,
703  * e.g. a ifdef that evaluates to false.
704  */
706  int linenum_;
707 
708  /** True iff we are currently parsing a macros content, otherwise false. */
710 
711  std::string read_word();
712  std::string read_line();
713  std::string read_rest_of_line();
714 
715  void skip_spaces();
716  void skip_eol();
718  void pop_token();
719  void put(char);
720  void put(const std::string& /*, int change_line = 0 */);
721  void conditional_skip(bool skip);
722 
723 public:
726  const std::string& history,
727  const std::string& name,
728  int line,
729  const std::string& dir,
730  const std::string& domain,
731  std::unique_ptr<std::map<std::string, std::string>> defines,
732  bool is_define = false);
733 
734  virtual bool get_chunk() override;
735 
736  virtual preprocessor::MODE parse_mode() override
737  {
739  }
740 
745 };
746 
748 {
749  throw std::logic_error("don't compare tokens with characters");
750 }
751 
753 {
754  return rhs == lhs;
755 }
756 
758 {
759  return !(lhs == rhs);
760 }
761 
763 {
764  return rhs != lhs;
765 }
766 
767 /** preprocessor_file constructor. */
768 preprocessor_file::preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index)
769  : preprocessor(t)
770  , files_()
771  , pos_()
772  , end_()
773  , name_(name)
774  , is_directory_(filesystem::is_directory(name))
775 {
776  if(is_directory_) {
777  filesystem::get_files_in_dir(name, &files_, nullptr,
781  );
782 
783  for(const std::string& fname : files_) {
784  std::size_t cpos = fname.rfind(" ");
785 
786  if(cpos != std::string::npos && cpos >= symbol_index) {
787  std::stringstream ss;
788  ss << "Found filename containing whitespace: '" << filesystem::base_name(fname)
789  << "' in included directory '" << name << "'.\nThe included symbol probably looks similar to '"
790  << filesystem::directory_name(fname.substr(symbol_index)) << "'";
791 
792  // TODO: find a real linenumber
793  parent_.error(ss.str(), -1);
794  }
795  }
796  } else {
797  // Handled in the init() function.
798  }
799 
800  pos_ = files_.begin();
801  end_ = files_.end();
802 }
803 
805 {
806  if(is_directory_) {
807  return;
808  }
809 
811 
812  if(!file_stream->good()) {
813  ERR_PREPROC << "Could not open file " << name_;
814  } else {
815  parent_.add_preprocessor<preprocessor_data>(std::move(file_stream), "",
818  );
819  }
820 }
821 
824  const std::string& history,
825  const std::string& name,
826  int linenum,
827  const std::string& directory,
828  const std::string& domain,
829  std::unique_ptr<std::map<std::string, std::string>> defines,
830  bool is_define)
831  : preprocessor(t)
832  , in_scope_(std::move(i))
833  , in_(*in_scope_)
834  , directory_(directory)
835  , strings_()
836  , local_defines_(std::move(defines))
837  , tokens_()
838  , slowpath_(0)
839  , skipping_(0)
840  , linenum_(linenum)
841  , is_define_(is_define)
842 {
843  std::ostringstream s;
844  s << history;
845 
846  if(!name.empty()) {
847  if(!history.empty()) {
848  s << ' ';
849  }
850 
851  s << get_file_code(name);
852  }
853 
854  if(!t.location_.empty()) {
855  s << ' ' << t.linenum_ << ' ' << t.location_;
856  }
857 
858  t.location_ = s.str();
859  t.linenum_ = linenum;
860 
861  t.buffer_ << INLINED_PREPROCESS_DIRECTIVE_CHAR << "line " << linenum << ' ' << t.location_ << '\n';
862 
863  if(t.textdomain_ != domain) {
864  t.buffer_ << INLINED_PREPROCESS_DIRECTIVE_CHAR << "textdomain " << domain << '\n';
865  t.textdomain_ = domain;
866  }
867 
869 }
870 
872 {
873  tokens_.emplace_back(t, strings_.size(), linenum_);
874 
876  // Macro expansions do not have any associated storage at start.
877  return;
879  /* Quoted strings are always inlined in the parent token. So
880  * they need neither storage nor metadata, unless the parent
881  * token is a macro expansion.
882  */
883  token_desc::token_type& outer_type = tokens_[tokens_.size() - 2].type;
884  if(outer_type != token_desc::token_type::macro_space) {
885  return;
886  }
887 
889  tokens_.back().stack_pos = strings_.size() + 1;
890  }
891 
892  std::ostringstream s;
893  if(!skipping_ && slowpath_) {
894  s << INLINED_PREPROCESS_DIRECTIVE_CHAR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
895  << INLINED_PREPROCESS_DIRECTIVE_CHAR << "textdomain " << parent_.textdomain_ << '\n';
896  }
897 
898  strings_.push_back(s.str());
899 }
900 
902 {
903  token_desc::token_type inner_type = tokens_.back().type;
904  unsigned stack_pos = tokens_.back().stack_pos;
905 
906  tokens_.pop_back();
907 
908  token_desc::token_type& outer_type = tokens_.back().type;
909 
910  if(inner_type == token_desc::token_type::macro_parens) {
911  // Parenthesized macro arguments are left on the stack.
912  assert(outer_type == token_desc::token_type::macro_space);
913  return;
914  }
915 
916  if(inner_type == token_desc::token_type::string || inner_type == token_desc::token_type::verbatim) {
917  // Quoted strings are always inlined.
918  assert(stack_pos == strings_.size());
919  return;
920  }
921 
922  if(outer_type == token_desc::token_type::macro_space) {
923  /* A macro expansion does not have any associated storage.
924  * Instead, storage of the inner token is not discarded
925  * but kept as a new macro argument. But if the inner token
926  * was a macro expansion, it is about to be appended, so
927  * prepare for it.
928  */
930  strings_.erase(strings_.begin() + stack_pos, strings_.end());
931  strings_.emplace_back();
932  }
933 
934  assert(stack_pos + 1 == strings_.size());
936 
937  return;
938  }
939 
940  strings_.erase(strings_.begin() + stack_pos, strings_.end());
941 }
942 
944 {
945  while(true) {
946  int c = in_.peek();
947 
948  if(in_.eof() || (c != ' ' && c != '\t')) {
949  return;
950  }
951 
952  in_.get();
953  }
954 }
955 
957 {
958  while(true) {
959  int c = in_.get();
960 
961  if(c == '\n') {
962  ++linenum_;
963  return;
964  }
965 
966  if(in_.eof()) {
967  return;
968  }
969  }
970 }
971 
973 {
974  std::string res;
975 
976  while(true) {
977  int c = in_.peek();
978 
979  if(c == preprocessor_streambuf::traits_type::eof() || utils::portable_isspace(c)) {
980  // DBG_PREPROC << "(" << res << ")";
981  return res;
982  }
983 
984  in_.get();
985  res += static_cast<char>(c);
986  }
987 }
988 
990 {
991  std::string res;
992 
993  while(true) {
994  int c = in_.get();
995 
996  if(c == '\n') {
997  ++linenum_;
998  return res;
999  }
1000 
1001  if(in_.eof()) {
1002  return res;
1003  }
1004 
1005  if(c != '\r') {
1006  res += static_cast<char>(c);
1007  }
1008  }
1009 }
1010 
1012 {
1013  std::string res;
1014 
1015  while(in_.peek() != '\n' && !in_.eof()) {
1016  int c = in_.get();
1017 
1018  if(c != '\r') {
1019  res += static_cast<char>(c);
1020  }
1021  }
1022 
1023  return res;
1024 }
1025 
1027 {
1028  if(skipping_) {
1029  return;
1030  }
1031 
1032  if(slowpath_) {
1033  strings_.back() += c;
1034  return;
1035  }
1036 
1037  int cond_linenum = c == '\n' ? linenum_ - 1 : linenum_;
1038 
1039  if(unsigned diff = cond_linenum - parent_.linenum_) {
1040  parent_.linenum_ = cond_linenum;
1041 
1042  if(diff <= parent_.location_.size() + 11) {
1043  parent_.buffer_ << std::string(diff, '\n');
1044  } else {
1046  }
1047  }
1048 
1049  if(c == '\n') {
1050  ++parent_.linenum_;
1051  }
1052 
1053  parent_.buffer_ << c;
1054 }
1055 
1056 void preprocessor_data::put(const std::string& s /*, int line_change*/)
1057 {
1058  if(skipping_) {
1059  return;
1060  }
1061 
1062  if(slowpath_) {
1063  strings_.back() += s;
1064  return;
1065  }
1066 
1067  parent_.buffer_ << s;
1068  // parent_.linenum_ += line_change;
1069 }
1070 
1072 {
1073  if(skip) {
1074  ++skipping_;
1075  }
1076 
1078 }
1079 
1081 {
1082  char c = static_cast<char>(in_.get());
1083  token_desc& token = tokens_.back();
1084 
1085  if(in_.eof()) {
1086  // The end of file was reached.
1087  // Make sure we don't have any incomplete tokens.
1088  char const* s;
1089 
1090  switch(token.type) {
1092  return false; // everything is fine
1097  s = "#ifdef or #ifndef";
1098  break;
1100  s = "Quoted string";
1101  break;
1103  s = "Verbatim string";
1104  break;
1107  s = "Macro substitution";
1108  break;
1110  s = "Macro argument";
1111  break;
1112  default:
1113  s = "???";
1114  }
1115 
1116  parent_.error(std::string(s) + " not terminated", token.linenum);
1117  }
1118 
1119  if(c == '\n') {
1120  ++linenum_;
1121  }
1122 
1123  if(c == static_cast<char>(INLINED_PREPROCESS_DIRECTIVE_CHAR)) {
1124  std::string buffer(1, c);
1125 
1126  while(true) {
1127  char d = static_cast<char>(in_.get());
1128 
1129  if(in_.eof() || d == '\n') {
1130  break;
1131  }
1132 
1133  buffer += d;
1134  }
1135 
1136  buffer += '\n';
1137  // line_change = 1-1 = 0
1138  put(buffer);
1140  put(c);
1141 
1142  if(c == '>' && in_.peek() == '>') {
1143  put(in_.get());
1144  pop_token();
1145  }
1146  } else if(c == '<' && in_.peek() == '<') {
1147  in_.get();
1149  put('<');
1150  put('<');
1151  } else if(c == '"') {
1153  parent_.quoted_ = false;
1154  put(c);
1155  pop_token();
1156  } else if(!parent_.quoted_) {
1157  parent_.quoted_ = true;
1159  put(c);
1160  } else {
1161  parent_.error("Nested quoted string", linenum_);
1162  }
1163  } else if(c == '{') {
1165  ++slowpath_;
1166  } else if(c == ')' && token.type == token_desc::token_type::macro_parens) {
1167  pop_token();
1168  } else if(c == '#' && !parent_.quoted_) {
1169  std::string command = read_word();
1170  bool comment = false;
1171 
1172  if(command == "define") {
1173  skip_spaces();
1174  int linenum = linenum_;
1175  std::vector<std::string> items = utils::split(read_line(), ' ');
1176  std::map<std::string, std::string> optargs;
1177 
1178  if(items.empty()) {
1179  parent_.error("No macro name found after #define directive", linenum);
1180  }
1181 
1182  std::string symbol = items.front();
1183  items.erase(items.begin());
1184  int found_arg = 0, found_enddef = 0, found_deprecate = 0;
1185  utils::optional<DEP_LEVEL> deprecation_level;
1186  std::string buffer, deprecation_detail;
1187  version_info deprecation_version = game_config::wesnoth_version;
1188  while(true) {
1189  if(in_.eof())
1190  break;
1191  char d = static_cast<char>(in_.get());
1192  if(d == '\n')
1193  ++linenum_;
1194  buffer += d;
1195  if(d == '#') {
1196  if(in_.peek() == 'a') {
1197  found_arg = 1;
1198  } else if(in_.peek() == 'd') {
1199  found_deprecate = 1;
1200  } else {
1201  found_enddef = 1;
1202  }
1203  } else {
1204  if(found_arg > 0 && ++found_arg == 4) {
1205  if(std::equal(buffer.end() - 3, buffer.end(), "arg")) {
1206  buffer.erase(buffer.end() - 4, buffer.end());
1207 
1208  skip_spaces();
1209  std::string argname = read_word();
1210  skip_eol();
1211 
1212  std::string argbuffer;
1213 
1214  int found_endarg = 0;
1215  while(true) {
1216  if(in_.eof()) {
1217  break;
1218  }
1219 
1220  char e = static_cast<char>(in_.get());
1221  if(e == '\n') {
1222  ++linenum_;
1223  }
1224 
1225  argbuffer += e;
1226 
1227  if(e == '#') {
1228  found_endarg = 1;
1229  } else if(found_endarg > 0 && ++found_endarg == 7) {
1230  if(std::equal(argbuffer.end() - 6, argbuffer.end(), "endarg")) {
1231  argbuffer.erase(argbuffer.end() - 7, argbuffer.end());
1232  optargs[argname] = argbuffer;
1233  skip_eol();
1234  break;
1235  } else {
1236  parent_.error("Unterminated #arg definition", linenum_);
1237  }
1238  }
1239  }
1240  }
1241  }
1242 
1243  if(found_deprecate > 0 && ++found_deprecate == 11) {
1244  if(std::equal(buffer.end() - 10, buffer.end(), "deprecated")) {
1245  buffer.erase(buffer.end() - 11, buffer.end());
1246  skip_spaces();
1247  try {
1249  if(deprecation_level) {
1250  deprecation_level = std::max(*deprecation_level, level);
1251  } else {
1252  deprecation_level = level;
1253  }
1254  } catch(const std::invalid_argument&) {
1255  // Meh, fall back to default of PREEMPTIVE...
1256  deprecation_level = DEP_LEVEL::PREEMPTIVE;
1257  }
1258  deprecation_version = game_config::wesnoth_version;
1259  if(deprecation_level == DEP_LEVEL::PREEMPTIVE || deprecation_level == DEP_LEVEL::FOR_REMOVAL) {
1260  skip_spaces();
1261  deprecation_version = std::max(deprecation_version, version_info(read_word()));
1262  }
1263  skip_spaces();
1264  if(!deprecation_detail.empty()){
1265  deprecation_detail += '\n';
1266  }
1267  deprecation_detail += read_rest_of_line();
1268  skip_eol();
1269  }
1270  }
1271 
1272  if(found_enddef > 0 && ++found_enddef == 7) {
1273  if(std::equal(buffer.end() - 6, buffer.end(), "enddef")) {
1274  break;
1275  } else {
1276  found_enddef = 0;
1277  if(std::equal(buffer.end() - 6, buffer.end(), "define")) { // TODO: Maybe add support for
1278  // this? This would fill feature
1279  // request #21343
1280  parent_.error(
1281  "Preprocessor error: #define is not allowed inside a #define/#enddef pair",
1282  linenum);
1283  }
1284  }
1285  }
1286  }
1287  }
1288 
1289  if(found_enddef != 7) {
1290  parent_.error("Unterminated preprocessor definition", linenum_);
1291  }
1292 
1293  if(!skipping_) {
1294  preproc_map::const_iterator old_i = parent_.defines_.find(symbol);
1295  if(old_i != parent_.defines_.end()) {
1296  std::ostringstream new_pos, old_pos;
1297  const preproc_define& old_d = old_i->second;
1298 
1299  new_pos << linenum << ' ' << parent_.location_;
1300  old_pos << old_d.linenum << ' ' << old_d.location;
1301 
1302  WRN_PREPROC << "Redefining macro " << symbol << " without explicit #undef at "
1303  << lineno_string(new_pos.str()) << '\n'
1304  << "previously defined at " << lineno_string(old_pos.str());
1305  }
1306 
1307  buffer.erase(buffer.end() - 7, buffer.end());
1308  parent_.defines_.insert_or_assign(symbol,
1309  preproc_define(buffer, items, optargs, parent_.textdomain_, linenum, parent_.location_,
1310  deprecation_detail, deprecation_level, deprecation_version));
1311 
1312  LOG_PREPROC << "defining macro " << symbol << " (location " << get_location(parent_.location_) << ")";
1313  }
1314  } else if(command == "ifdef" || command == "ifndef") {
1315  const bool negate = command[2] == 'n';
1316  skip_spaces();
1317  const std::string& symbol = read_word();
1318  if(symbol.empty()) {
1319  parent_.error("No macro argument found after #ifdef/#ifndef directive", linenum_);
1320  }
1321  bool found = parent_.defines_.count(symbol) != 0;
1322  DBG_PREPROC << "testing for macro " << symbol << ": " << (found ? "defined" : "not defined");
1323  conditional_skip(negate ? found : !found);
1324  } else if(command == "ifhave" || command == "ifnhave") {
1325  const bool negate = command[2] == 'n';
1326  skip_spaces();
1327  const std::string& symbol = read_word();
1328  if(symbol.empty()) {
1329  parent_.error("No path argument found after #ifhave/#ifnhave directive", linenum_);
1330  }
1331  bool found = filesystem::get_wml_location(symbol, directory_).has_value();
1332  DBG_PREPROC << "testing for file or directory " << symbol << ": " << (found ? "found" : "not found");
1333  conditional_skip(negate ? found : !found);
1334  } else if(command == "ifver" || command == "ifnver") {
1335  const bool negate = command[2] == 'n';
1336 
1337  skip_spaces();
1338  const std::string& vsymstr = read_word();
1339  skip_spaces();
1340  const std::string& vopstr = read_word();
1341  skip_spaces();
1342  const std::string& vverstr = read_word();
1343 
1344  const VERSION_COMP_OP vop = parse_version_op(vopstr);
1345 
1346  if(vop == OP_INVALID) {
1347  parent_.error("Invalid #ifver/#ifnver operator", linenum_);
1348  } else if(parent_.defines_.count(vsymstr) != 0) {
1349  const preproc_define& sym = parent_.defines_[vsymstr];
1350 
1351  if(!sym.arguments.empty()) {
1352  parent_.error("First argument macro in #ifver/#ifnver should not require arguments", linenum_);
1353  }
1354 
1355  version_info const version1(sym.value);
1356  version_info const version2(vverstr);
1357 
1358  const bool found = do_version_check(version1, vop, version2);
1359  DBG_PREPROC << "testing version '" << version1.str() << "' against '" << version2.str() << "' ("
1360  << vopstr << "): " << (found ? "match" : "no match");
1361 
1362  conditional_skip(negate ? found : !found);
1363  } else {
1364  std::string err = "Undefined macro in #ifver/#ifnver first argument: '";
1365  err += vsymstr;
1366  err += "'";
1368  }
1369  } else if(command == "else") {
1371  pop_token();
1372  --skipping_;
1375  pop_token();
1376  ++skipping_;
1378  } else {
1379  parent_.error("Unexpected #else", linenum_);
1380  }
1381  } else if(command == "endif") {
1382  switch(token.type) {
1385  --skipping_;
1388  break;
1389  default:
1390  parent_.error("Unexpected #endif", linenum_);
1391  }
1392  pop_token();
1393  } else if(command == "textdomain") {
1394  skip_spaces();
1395  const std::string& s = read_word();
1396  if(s != parent_.textdomain_) {
1397  put("#textdomain ");
1398  put(s);
1399  parent_.textdomain_ = s;
1400  }
1401  comment = true;
1402  } else if(command == "enddef") {
1403  parent_.error("Unexpected #enddef", linenum_);
1404  } else if(command == "undef") {
1405  skip_spaces();
1406  const std::string& symbol = read_word();
1407  if(!skipping_) {
1408  parent_.defines_.erase(symbol);
1409  LOG_PREPROC << "undefine macro " << symbol << " (location " << get_location(parent_.location_) << ")";
1410  }
1411  } else if(command == "error") {
1412  if(!skipping_) {
1413  skip_spaces();
1414  std::ostringstream error;
1415  error << "#error: \"" << read_rest_of_line() << '"';
1416  parent_.error(error.str(), linenum_);
1417  } else
1418  DBG_PREPROC << "Skipped an error";
1419  } else if(command == "warning") {
1420  if(!skipping_) {
1421  skip_spaces();
1422  std::ostringstream warning;
1423  warning << "#warning: \"" << read_rest_of_line() << '"';
1424  parent_.warning(warning.str(), linenum_);
1425  } else {
1426  DBG_PREPROC << "Skipped a warning";
1427  }
1428  } else if(command == "deprecated") {
1429  // The current file is deprecated, so print a message
1430  skip_spaces();
1432  try {
1434  } catch(const std::invalid_argument&) {
1435  // Meh, just fall back to the default of PREEMPTIVE...
1436  }
1439  skip_spaces();
1440  version = version_info(read_word());
1441  }
1442  skip_spaces();
1443  std::string detail = read_rest_of_line();
1445  } else {
1447  }
1448 
1449  skip_eol();
1450  if(comment) {
1451  put('\n');
1452  }
1454  if(c == '(') {
1455  // If a macro argument was started, it is implicitly ended.
1458  } else if(utils::portable_isspace(c)) {
1459  // If a macro argument was started, it is implicitly ended.
1461  } else if(c == '}') {
1462  --slowpath_;
1463  if(skipping_) {
1464  pop_token();
1465  return true;
1466  }
1467 
1468  // FIXME: is this obsolete?
1469  // if (token.type == token_desc::MACRO_SPACE) {
1470  // if (!strings_.back().empty()) {
1471  // std::ostringstream error;
1472  // std::ostringstream location;
1473  // error << "Can't parse new macro parameter with a macro call scope open";
1474  // location<<linenum_<<' '<<parent_.location_;
1475  // parent_.error(error.str(), location.str());
1476  // }
1477  // strings_.pop_back();
1478  //}
1479 
1480  if(strings_.size() <= static_cast<std::size_t>(token.stack_pos)) {
1481  parent_.error("No macro or file substitution target specified", linenum_);
1482  }
1483 
1484  std::string symbol = strings_[token.stack_pos];
1485  std::string::size_type pos;
1486  while((pos = symbol.find(INLINED_PREPROCESS_DIRECTIVE_CHAR)) != std::string::npos) {
1487  std::string::iterator b = symbol.begin(); // invalidated at each iteration
1488  symbol.erase(b + pos, b + symbol.find('\n', pos + 1) + 1);
1489  }
1490 
1491  std::map<std::string, std::string>::const_iterator arg;
1492  preproc_map::const_iterator macro;
1493 
1494  // If this is a known pre-processing symbol, then we insert it,
1495  // otherwise we assume it's a file name to load.
1496  if(symbol == current_file_str && strings_.size() - token.stack_pos == 1) {
1497  pop_token();
1499  } else if(symbol == current_dir_str && strings_.size() - token.stack_pos == 1) {
1500  pop_token();
1502  } else if(symbol == left_curly_str && strings_.size() - token.stack_pos == 1) {
1503  pop_token();
1504  put("{");
1505  } else if(symbol == right_curly_str && strings_.size() - token.stack_pos == 1) {
1506  pop_token();
1507  put("}");
1508  } else if(local_defines_ && (arg = local_defines_->find(symbol)) != local_defines_->end()) {
1509  if(strings_.size() - token.stack_pos != 1) {
1510  std::ostringstream error;
1511  error << "Macro argument '" << symbol << "' does not expect any arguments";
1512  parent_.error(error.str(), linenum_);
1513  }
1514 
1515  std::ostringstream v;
1516  v << arg->second << INLINED_PREPROCESS_DIRECTIVE_CHAR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1517  << INLINED_PREPROCESS_DIRECTIVE_CHAR << "textdomain " << parent_.textdomain_ << '\n';
1518 
1519  pop_token();
1520  put(v.str());
1521  } else if(parent_.depth() < 100 && (macro = parent_.defines_.find(symbol)) != parent_.defines_.end()) {
1522  const preproc_define& val = macro->second;
1523  std::size_t nb_arg = strings_.size() - token.stack_pos - 1;
1524  std::size_t optional_arg_num = 0;
1525 
1526  auto defines = std::make_unique<std::map<std::string, std::string>>();
1527  const std::string& dir = filesystem::directory_name(val.location.substr(0, val.location.find(' ')));
1528 
1529  if(val.is_deprecated()) {
1531  }
1532 
1533  for(std::size_t i = 0; i < nb_arg; ++i) {
1534  if(i < val.arguments.size()) {
1535  // Normal mandatory arguments
1536 
1537  (*defines)[val.arguments[i]] = strings_[token.stack_pos + i + 1];
1538  } else {
1539  // These should be optional argument overrides
1540 
1541  std::string str = strings_[token.stack_pos + i + 1];
1542  std::size_t equals_pos = str.find_first_of("=");
1543 
1544  if(equals_pos != std::string::npos) {
1545  std::size_t argname_pos = str.substr(0, equals_pos).find_last_of(" \n") + 1;
1546 
1547  std::string argname = str.substr(argname_pos, equals_pos - argname_pos);
1548 
1549  if(val.optional_arguments.find(argname) != val.optional_arguments.end()) {
1550  (*defines)[argname] = str.substr(equals_pos + 1);
1551 
1552  optional_arg_num++;
1553 
1554  DBG_PREPROC << "Found override for " << argname << " in call to macro " << symbol;
1555  } else {
1556  std::ostringstream warning;
1557  warning << "Unrecognized optional argument passed to macro '" << symbol << "': '"
1558  << argname << "'";
1559  parent_.warning(warning.str(), linenum_);
1560 
1561  optional_arg_num++; // To prevent the argument number check from blowing up
1562  }
1563  }
1564  }
1565  }
1566 
1567  // If the macro definition has any optional arguments, insert their defaults
1568  if(val.optional_arguments.size() > 0) {
1569  for(const auto& argument : val.optional_arguments) {
1570  if(defines->find(argument.first) == defines->end()) {
1571  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1572 
1573  buf->textdomain_ = parent_.textdomain_;
1574  std::istream in(buf.get());
1575 
1576  filesystem::scoped_istream buffer{new std::istringstream(argument.second)};
1577 
1578  auto temp_defines = std::make_unique<std::map<std::string, std::string>>();
1579  temp_defines->insert(defines->begin(), defines->end());
1580 
1581  buf->add_preprocessor<preprocessor_data>(
1582  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(temp_defines), false);
1583 
1584  std::ostringstream res;
1585  res << in.rdbuf();
1586 
1587  DBG_PREPROC << "Setting default for optional argument " << argument.first << " in macro "
1588  << symbol;
1589 
1590  (*defines)[argument.first] = res.str();
1591  }
1592  }
1593  }
1594 
1595  if(nb_arg - optional_arg_num != val.arguments.size()) {
1596  const std::vector<std::string>& locations = utils::quoted_split(val.location, ' ');
1597  const std::string filename = locations.empty() ? "<command-line>" : get_filename(locations[0]);
1598  std::ostringstream error;
1599  error << "Preprocessor symbol '" << symbol << "' defined at " << filename << ":"
1600  << val.linenum << " expects " << val.arguments.size() << " arguments, but has "
1601  << nb_arg - optional_arg_num << " arguments";
1602  parent_.error(error.str(), linenum_);
1603  }
1604 
1605  filesystem::scoped_istream buffer{new std::istringstream(val.value)};
1606 
1607  pop_token();
1608 
1609  if(!slowpath_) {
1610  DBG_PREPROC << "substituting macro " << symbol;
1611 
1613  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1614  } else {
1615  DBG_PREPROC << "substituting (slow) macro " << symbol;
1616 
1617  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1618 
1619  // Make the nested preprocessor_data responsible for
1620  // restoring our current textdomain if needed.
1621  buf->textdomain_ = parent_.textdomain_;
1622 
1623  std::ostringstream res;
1624  {
1625  std::istream in(buf.get());
1626  buf->add_preprocessor<preprocessor_data>(
1627  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1628 
1629  res << in.rdbuf();
1630  }
1631 
1632  put(res.str());
1633  }
1634  } else if(parent_.depth() < 40) {
1635  LOG_PREPROC << "Macro definition not found for " << symbol << ", attempting to open as file.";
1636  pop_token();
1637 
1638  if(auto nfname = filesystem::get_wml_location(symbol, directory_)) {
1639  if(!slowpath_)
1640  // nfname.size() - symbol.size() gives you an index into nfname
1641  // This does not necessarily match the symbol though, as it can start with ~ or ./
1642  parent_.add_preprocessor<preprocessor_file>(nfname.value(), nfname->size() - symbol.size());
1643  else {
1644  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1645 
1646  std::ostringstream res;
1647  {
1648  std::istream in(buf.get());
1649  buf->add_preprocessor<preprocessor_file>(nfname.value(), nfname->size() - symbol.size());
1650 
1651  res << in.rdbuf();
1652  }
1653 
1654  put(res.str());
1655  }
1656  } else {
1657  std::ostringstream error;
1658  error << "Macro/file '" << symbol << "' is missing";
1659  parent_.error(error.str(), linenum_);
1660  }
1661  } else {
1662  parent_.error("Too many nested preprocessing inclusions", linenum_);
1663  }
1664  } else if(!skipping_) {
1666  std::ostringstream s;
1667  s << INLINED_PREPROCESS_DIRECTIVE_CHAR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1668  << INLINED_PREPROCESS_DIRECTIVE_CHAR << "textdomain " << parent_.textdomain_ << '\n';
1669 
1670  strings_.push_back(s.str());
1672  }
1673  put(c);
1674  }
1675  } else {
1676  put(c);
1677  }
1678 
1679  return true;
1680 }
1681 
1682 
1683 // ==================================================================================
1684 // PREPROCESSOR ISTREAM
1685 // ==================================================================================
1686 
1687 class preprocessor_istream : public std::istream
1688 {
1689 public:
1691  : std::istream(nullptr)
1692  , owned_defines_()
1694  {
1695  }
1696 
1698  : std::istream(nullptr)
1699  , owned_defines_()
1700  , buffer_(defines)
1701  {
1702  }
1703 
1705  {
1706  clear(std::ios_base::goodbit);
1707  exceptions(std::ios_base::goodbit);
1708  rdbuf(nullptr);
1709  }
1710 
1711  template<typename Preprocessor, typename... Args>
1712  void process(Args&&... args)
1713  {
1714  // Take ownership of the processing buffer.
1715  rdbuf(&buffer_);
1716 
1717  // Begin processing.
1718  buffer_.add_preprocessor<Preprocessor>(std::forward<Args>(args)...);
1719  }
1720 
1721 private:
1722  /** Defines map to be used if no external map is given. */
1724 
1725  /** The underlying processing buffer. */
1727 };
1728 
1729 // ==================================================================================
1730 // FREE-STANDING FUNCTIONS
1731 // ==================================================================================
1732 
1733 filesystem::scoped_istream preprocess_file(const std::string& fname, preproc_map& defines)
1734 {
1735  log_scope("preprocessing file " + fname + " ...");
1736 
1737  auto stream = std::make_unique<preprocessor_istream>(defines);
1738  stream->process<preprocessor_file>(fname);
1739  return stream;
1740 }
1741 
1743 {
1744  log_scope("preprocessing file " + fname + " ...");
1745 
1746  auto stream = std::make_unique<preprocessor_istream>();
1747  stream->process<preprocessor_file>(fname);
1748  return stream;
1749 }
1750 
1752  const std::string& contents, const std::string& textdomain)
1753 {
1754  log_scope("preprocessing string " + contents.substr(0, 10) + " ...");
1755 
1756  auto stream = std::make_unique<preprocessor_istream>();
1757  stream->process<preprocessor_data>(
1758  std::make_unique<std::istringstream>(contents), "<string>", "", 1, game_config::path, textdomain, nullptr);
1759 
1760  return stream;
1761 }
1762 
1764  const std::string& contents, const std::string& textdomain, preproc_map& defines)
1765 {
1766  log_scope("preprocessing string " + contents.substr(0, 10) + " ...");
1767 
1768  auto stream = std::make_unique<preprocessor_istream>(defines);
1769  stream->process<preprocessor_data>(
1770  std::make_unique<std::istringstream>(contents), "<string>", "", 1, game_config::path, textdomain, nullptr);
1771 
1772  return stream;
1773 }
1774 
1775 void preprocess_resource(const std::string& res_name,
1776  preproc_map* defines_map,
1777  bool write_cfg,
1778  bool write_plain_cfg,
1779  const std::string& parent_directory)
1780 {
1781  if(filesystem::is_directory(res_name)) {
1782  std::vector<std::string> dirs, files;
1783 
1786 
1787  // Subdirectories
1788  for(const std::string& dir : dirs) {
1789  LOG_PREPROC << "processing sub-dir: " << dir;
1790  preprocess_resource(dir, defines_map, write_cfg, write_plain_cfg, parent_directory);
1791  }
1792 
1793  // Files in current directory
1794  for(const std::string& file : files) {
1795  if(filesystem::is_cfg(file)) {
1796  preprocess_resource(file, defines_map, write_cfg, write_plain_cfg, parent_directory);
1797  }
1798  }
1799 
1800  return;
1801  }
1802 
1803  LOG_PREPROC << "processing resource: " << res_name;
1804 
1805  // disable filename encoding to get clear #line in cfg.plain
1806  encode_filename = false;
1807 
1808  filesystem::scoped_istream stream = defines_map
1809  ? preprocess_file(res_name, *defines_map)
1810  : preprocess_file(res_name);
1811 
1812  std::stringstream ss;
1813 
1814  // Set the failbit so if we get any preprocessor exceptions (e.g.:preproc_config::error)
1815  // they will be propagated in the main program, instead of just setting the
1816  // failbit on the stream. This was necessary in order for the MSVC and GCC
1817  // binaries to behave the same way.
1818  ss.exceptions(std::ios_base::failbit);
1819 
1820  ss << (*stream).rdbuf();
1821 
1822  LOG_PREPROC << "processing finished";
1823 
1824  if(write_cfg || write_plain_cfg) {
1825  std::string streamContent = ss.str();
1826  config cfg = io::read(streamContent);
1827 
1828  const std::string preproc_res_name = parent_directory + "/" + filesystem::base_name(res_name);
1829 
1830  // Write the processed cfg file
1831  if(write_cfg) {
1832  LOG_PREPROC << "writing cfg file: " << preproc_res_name;
1833 
1835  io::write(*filesystem::ostream_file(preproc_res_name), cfg);
1836  }
1837 
1838  // Write the plain cfg file
1839  if(write_plain_cfg) {
1840  LOG_PREPROC << "writing plain cfg file: " << (preproc_res_name + ".plain");
1841 
1843  filesystem::write_file(preproc_res_name + ".plain", streamContent);
1844  }
1845  }
1846 }
map_location loc
Definition: move.cpp:172
double t
Definition: astarsearch.cpp:63
unsigned in
If equal to search_counter, the node is off the list.
Definition: astarsearch.cpp:70
Helper class for buffering a std::istream.
Helper class for buffering a std::istream.
bool eof() const
Is the end of input reached?
int get()
Gets and consumes a character from the buffer.
int peek()
Gets a character from the buffer.
Class for writing a config out to a file in pieces.
void close_child(const std::string &key)
void write_key_val(const std::string &key, const T &value)
This template function will work with any type that can be assigned to an attribute_value.
void open_child(const std::string &key)
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:158
bool has_attribute(config_key_type key) const
Definition: config.cpp:157
child_itors child_range(config_key_type key)
Definition: config.cpp:268
optional_config_impl< config > optional_child(config_key_type key, int n=0)
Equivalent to mandatory_child, but returns an empty optional if the nth child was not found.
Definition: config.cpp:380
Specialized preprocessor for handling any kind of input stream.
std::unique_ptr< std::map< std::string, std::string > > local_defines_
Mapping of macro arguments to their content.
int skipping_
Non-zero when the preprocessor has to skip some input text.
virtual preprocessor::MODE parse_mode() override
Returns the appropriate parsing mode for this preprocessor.
void conditional_skip(bool skip)
int slowpath_
Set to true whenever input tokens cannot be directly sent to the target buffer.
std::string read_rest_of_line()
friend bool operator!=(preprocessor_data::token_desc::token_type, char)
std::vector< std::string > strings_
Buffer for delayed input processing.
filesystem::scoped_istream in_scope_
Manages the lifetime of the std::istream pointer we own.
preprocessor_data(preprocessor_streambuf &, filesystem::scoped_istream, const std::string &history, const std::string &name, int line, const std::string &dir, const std::string &domain, std::unique_ptr< std::map< std::string, std::string >> defines, bool is_define=false)
buffered_istream in_
Input stream.
friend bool operator==(preprocessor_data::token_desc::token_type, char)
bool is_define_
True iff we are currently parsing a macros content, otherwise false.
std::vector< token_desc > tokens_
Stack of nested preprocessing chunks.
std::string read_line()
std::string read_word()
virtual bool get_chunk() override
Preprocesses and sends some text to the parent_ buffer.
std::string directory_
void push_token(token_desc::token_type)
Specialized preprocessor for handling a file or a set of files.
std::vector< std::string >::const_iterator pos_
std::vector< std::string > files_
virtual bool get_chunk() override
Inserts and processes the next file in the list of included files.
const std::string & name_
std::vector< std::string >::const_iterator end_
virtual void init() override
Allows specifying any actions that need to be called after the constructor completes.
preprocessor_file(preprocessor_streambuf &t, const std::string &name, std::size_t symbol_index=-1)
Constructor.
preprocessor_istream(preproc_map &defines)
void process(Args &&... args)
preproc_map owned_defines_
Defines map to be used if no external map is given.
preprocessor_streambuf buffer_
The underlying processing buffer.
Target for sending preprocessed output.
std::stringstream buffer_
Buffer filled by the current preprocessor.
preprocessor * current() const
bool quoted_
Set to true if one preprocessor for this target started to read a string.
virtual int underflow() override
Inherited from basic_streambuf.
void restore_old_preprocessor()
Restores the old preprocessing context.
std::string out_buffer_
Buffer read by the STL stream.
std::string get_current_file()
Decodes the filenames placed in a location.
preprocessor_streambuf(preproc_map &def)
std::deque< std::unique_ptr< preprocessor > > preprocessor_queue_
Input preprocessor queue.
void add_preprocessor(A &&... args)
void error(const std::string &, int)
preprocessor_streambuf(const preprocessor_streambuf &t)
void warning(const std::string &, int)
Base class for preprocessing an input.
virtual MODE parse_mode()
Returns the appropriate parsing mode for this preprocessor.
std::string old_textdomain_
virtual void init()
Allows specifying any actions that need to be called after the constructor completes.
virtual ~preprocessor()
std::string old_location_
friend class preprocessor_streambuf
preprocessor(preprocessor_streambuf &t)
Sets up a new preprocessor for stream buffer t.
virtual bool get_chunk()=0
Preprocesses and sends some text to the parent_ buffer.
preprocessor_streambuf & parent_
Represents version numbers.
std::string str() const
Serializes the version number into string form.
Definitions for the interface to Wesnoth Markup Language (WML).
std::string deprecated_message(const std::string &elem_name, DEP_LEVEL level, const version_info &version, const std::string &detail)
Definition: deprecation.cpp:29
DEP_LEVEL
See https://wiki.wesnoth.org/CompatibilityStandards for more info.
Definition: deprecation.hpp:21
const config * cfg
std::size_t i
Definition: function.cpp:1032
bool do_version_check(const version_info &a, VERSION_COMP_OP op, const version_info &b)
VERSION_COMP_OP parse_version_op(const std::string &op_str)
Interfaces for manipulating version numbers of engine, add-ons, etc.
VERSION_COMP_OP
@ OP_INVALID
Standard logging facilities (interface).
#define log_scope(description)
Definition: log.hpp:275
void clear()
Clear the current render target.
Definition: draw.cpp:42
void line(int from_x, int from_y, int to_x, int to_y)
Draw a line.
Definition: draw.cpp:189
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, name_mode mode, filter_mode filter, reorder_mode reorder, file_tree_checksum *checksum)
Get a list of all files and/or directories in a given directory.
Definition: filesystem.cpp:463
bool is_cfg(const std::string &filename)
Returns true if the file ends with the wmlfile extension.
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
bool is_directory(const std::string &fname)
Returns true if the given file is a directory.
utils::optional< std::string > get_wml_location(const std::string &path, const utils::optional< std::string > &current_dir)
Returns a translated path to the actual file or directory, if it exists.
filesystem::scoped_ostream ostream_file(const std::string &fname, std::ios_base::openmode mode, bool create_directory)
static bool create_directory_if_missing_recursive(const bfs::path &dirpath)
Definition: filesystem.cpp:399
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:53
void write_file(const std::string &fname, const std::string &data, std::ios_base::openmode mode)
Throws io_exception if an error occurs.
std::string get_short_wml_path(const std::string &filename)
Returns a short path to filename, skipping the (user) data directory.
std::string directory_name(const std::string &file)
Returns the directory name of a file, with filename stripped.
std::string path
Definition: filesystem.cpp:106
const version_info wesnoth_version(VERSION)
config read(std::istream &in, abstract_validator *validator)
Definition: parser.cpp:600
void write(std::ostream &out, const configr_of &cfg, unsigned int level)
Definition: parser.cpp:737
logger & err()
Definition: log.cpp:339
std::vector< std::string > quoted_split(const std::string &val, char c, int flags, char quote)
This function is identical to split(), except it does not split when it otherwise would if the previo...
int stoi(std::string_view str)
Same interface as std::stoi and meant as a drop in replacement, except:
Definition: charconv.hpp:155
bool portable_isspace(const char c)
std::string escape(std::string_view str, const char *special_chars)
Prepends a configurable set of characters with a backslash.
std::vector< std::string > split(const config_attribute_value &val)
std::string::const_iterator iterator
Definition: tokenizer.hpp:25
#define DBG_PREPROC
static lg::log_domain log_preprocessor("preprocessor")
filesystem::scoped_istream preprocess_string(const std::string &contents, const std::string &textdomain)
static const std::string left_curly_str
static const std::string current_dir_str
#define ERR_PREPROC
std::string lineno_string(const std::string &lineno)
void preprocess_resource(const std::string &res_name, preproc_map *defines_map, bool write_cfg, bool write_plain_cfg, const std::string &parent_directory)
#define WRN_PREPROC
static std::string get_file_code(const std::string &filename)
static bool encode_filename
static const std::string current_file_str
bool operator==(preprocessor_data::token_desc::token_type, char)
static std::string get_filename(const std::string &file_code)
static std::string get_location(const std::string &loc)
std::ostream & operator<<(std::ostream &stream, const preproc_define &def)
#define LOG_PREPROC
static std::map< std::string, int > file_number_map
static const std::string right_curly_str
bool operator!=(preprocessor_data::token_desc::token_type rhs, char lhs)
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map &defines)
Function to use the WML preprocessor on a file.
static std::string preprocessor_error_detail_prefix
std::map< std::string, struct preproc_define > preproc_map
constexpr unsigned char INLINED_PREPROCESS_DIRECTIVE_CHAR
Definition: tokenizer.hpp:29
std::string filename
Filename.
void write(config_writer &, const std::string &) const
std::string value
version_info deprecation_version
bool operator<(const preproc_define &) const
std::string textdomain
void write_argument(config_writer &, const std::string &) const
void read(const config &)
void read_argument(const config &)
std::string deprecation_message
std::vector< std::string > arguments
static void insert(preproc_map &, const config &)
std::map< std::string, std::string > optional_arguments
bool operator==(const preproc_define &) const
utils::optional< DEP_LEVEL > deprecation_level
std::string location
bool is_deprecated() const
Description of a preprocessing chunk.
token_desc(token_type type, const int stack_pos, const int linenum)
int stack_pos
Starting position in strings_ of the delayed text for this chunk.
contains the current text being parsed as well as the token_type of what's being parsed.
Definition: tokenizer.hpp:41
token_type type
Definition: tokenizer.hpp:83
mock_char c
mock_party p
static map_location::direction n
static map_location::direction s
Some defines: VERSION, PACKAGE, MIN_SAVEGAME_VERSION.
#define PACKAGE
Definition: wesconfig.h:23
#define d
#define e
#define b