The Battle for Wesnoth  1.19.14+dev
preprocessor.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2005 - 2025
3  by Guillaume Melquiond <guillaume.melquiond@gmail.com>
4  Copyright (C) 2003 by David White <dave@whitevine.net>
5  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
6 
7  This program is free software; you can redistribute it and/or modify
8  it under the terms of the GNU General Public License as published by
9  the Free Software Foundation; either version 2 of the License, or
10  (at your option) any later version.
11  This program is distributed in the hope that it will be useful,
12  but WITHOUT ANY WARRANTY.
13 
14  See the COPYING file for more details.
15 */
16 
17 /**
18  * @file
19  * WML preprocessor.
20  */
21 
23 
24 #include "buffered_istream.hpp"
25 #include "config.hpp"
26 #include "log.hpp"
28 #include "serialization/parser.hpp"
30 #include "game_version.hpp"
31 #include "wesconfig.h"
32 #include "deprecation.hpp"
33 
34 #include <stdexcept>
35 #include <deque>
36 
37 static lg::log_domain log_preprocessor("preprocessor");
38 #define ERR_PREPROC LOG_STREAM(err, log_preprocessor)
39 #define WRN_PREPROC LOG_STREAM(warn, log_preprocessor)
40 #define LOG_PREPROC LOG_STREAM(info, log_preprocessor)
41 #define DBG_PREPROC LOG_STREAM(debug, log_preprocessor)
42 
43 static const std::string current_file_str = "CURRENT_FILE";
44 static const std::string current_dir_str = "CURRENT_DIRECTORY";
45 static const std::string left_curly_str = "LEFT_BRACE";
46 static const std::string right_curly_str = "RIGHT_BRACE";
47 
48 // map associating each filename encountered to a number
49 static std::map<std::string, int> file_number_map;
50 
51 static bool encode_filename = true;
52 
53 static std::string preprocessor_error_detail_prefix = "\n ";
54 
55 // get filename associated to this code
56 static std::string get_filename(const std::string& file_code)
57 {
58  if(!encode_filename) {
59  return file_code;
60  }
61 
62  std::stringstream s;
63  s << file_code;
64  int n = 0;
65  s >> std::hex >> n;
66 
67  for(const auto& p : file_number_map) {
68  if(p.second == n) {
69  return p.first;
70  }
71  }
72 
73  return "<unknown>";
74 }
75 
76 // Get code associated to this filename
77 static std::string get_file_code(const std::string& filename)
78 {
79  if(!encode_filename) {
80  return filename;
81  }
82 
83  // Current number of encountered filenames
84  static int current_file_number = 0;
85 
86  int& fnum = file_number_map[utils::escape(filename, " \\")];
87  if(fnum == 0) {
88  fnum = ++current_file_number;
89  }
90 
91  std::ostringstream shex;
92  shex << std::hex << fnum;
93 
94  return shex.str();
95 }
96 
97 // decode the filenames placed in a location
98 static std::string get_location(const std::string& loc)
99 {
100  std::string res;
101  std::vector<std::string> pos = utils::quoted_split(loc, ' ');
102 
103  if(pos.empty()) {
104  return res;
105  }
106 
107  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
108  while(true) {
109  res += get_filename(*(i++));
110 
111  if(i == end) {
112  break;
113  }
114 
115  res += ' ';
116  res += *(i++);
117 
118  if(i == end) {
119  break;
120  }
121 
122  res += ' ';
123  }
124 
125  return res;
126 }
127 
128 
129 // ==================================================================================
130 // PREPROC_DEFINE IMPLEMENTATION
131 // ==================================================================================
132 
134 {
135  return value == v.value && arguments == v.arguments;
136 }
137 
139 {
140  if(location < v.location) {
141  return true;
142  }
143 
144  if(v.location < location) {
145  return false;
146  }
147 
148  return linenum < v.linenum;
149 }
150 
151 void preproc_define::write_argument(config_writer& writer, const std::string& arg) const
152 {
153  const std::string key = "argument";
154 
155  writer.open_child(key);
156 
157  writer.write_key_val("name", arg);
158  writer.close_child(key);
159 }
160 
161 void preproc_define::write_argument(config_writer& writer, const std::string& arg, const std::string& default_value) const
162 {
163  const std::string key = "argument";
164 
165  writer.open_child(key);
166 
167  writer.write_key_val("name", arg);
168  writer.write_key_val("default", default_value);
169  writer.close_child(key);
170 }
171 
172 void preproc_define::write(config_writer& writer, const std::string& name) const
173 {
174  const std::string key = "preproc_define";
175  writer.open_child(key);
176 
177  writer.write_key_val("name", name);
178  writer.write_key_val("value", value);
179  writer.write_key_val("textdomain", textdomain);
180  writer.write_key_val("linenum", std::to_string(linenum));
181  writer.write_key_val("location", get_location(location));
182 
183  if(is_deprecated()) {
184  writer.open_child("deprecated");
185  writer.write_key_val("level", int(*deprecation_level));
186  writer.write_key_val("version", deprecation_version.str());
187  writer.write_key_val("message", deprecation_message);
188  writer.close_child("deprecated");
189  }
190 
191  for(const std::string& arg : arguments) {
192  write_argument(writer, arg);
193  }
194 
195  for(const auto& [key, default_value] : optional_arguments) {
196  write_argument(writer, key, default_value);
197  }
198 
199  writer.close_child(key);
200 }
201 
203 {
204  if(cfg.has_attribute("default")) {
205  optional_arguments.emplace(cfg["name"], cfg["default"]);
206  } else {
207  arguments.push_back(cfg["name"]);
208  }
209 }
210 
212 {
213  value = cfg["value"].str();
214  textdomain = cfg["textdomain"].str();
215  linenum = cfg["linenum"].to_int();
216  location = cfg["location"].str();
217 
218  if(auto deprecated = cfg.optional_child("deprecated")) {
219  deprecation_level = DEP_LEVEL(deprecated.value()["level"].to_int());
220  deprecation_version = deprecated.value()["version"].str();
221  deprecation_message = deprecated.value()["message"].str();
222  }
223 
224  for(const config& arg : cfg.child_range("argument")) {
225  read_argument(arg);
226  }
227 }
228 
230 {
231  map.try_emplace(cfg["name"], cfg);
232 }
233 
234 std::ostream& operator<<(std::ostream& stream, const preproc_define& def)
235 {
236  return stream << "value: " << def.value << " arguments: " << def.location;
237 }
238 
239 std::ostream& operator<<(std::ostream& stream, const preproc_map::value_type& def)
240 {
241  return stream << def.second;
242 }
243 
244 // ==================================================================================
245 // PREPROCESSOR BASE
246 // ==================================================================================
247 
249 
250 /**
251  * Base class for preprocessing an input.
252  */
254 {
256 
257 protected:
258  /**
259  * Sets up a new preprocessor for stream buffer \a t.
260  * Saves the current preprocessing context of #parent_. It will be automatically restored on destruction.
261  *
262  * It relies on preprocessor_streambuf so it's implemented after that class is declared.
263  */
265 
267 
268 public:
269  virtual ~preprocessor()
270  {
271  }
272 
273  /** Allows specifying any actions that need to be called after the constructor completes. */
274  virtual void init()
275  {
276  }
277 
278  /**
279  * Preprocesses and sends some text to the #parent_ buffer.
280  * @return false when the input has no data left.
281  */
282  virtual bool get_chunk() = 0;
283 
285 
286  /** Returns the appropriate parsing mode for this preprocessor. */
287  virtual MODE parse_mode()
288  {
289  return NO_PARSING;
290  }
291 
292 private:
293  std::string old_textdomain_;
294  std::string old_location_;
295 
297 };
298 
299 
300 // ==================================================================================
301 // PREPROCESSOR BUFFER
302 // ==================================================================================
303 
304 /**
305  * Target for sending preprocessed output.
306  * Objects of this class can be plugged into an STL stream.
307  */
308 class preprocessor_streambuf : public std::streambuf
309 {
310 public:
312  : std::streambuf()
313  , out_buffer_("")
314  , buffer_()
316  , defines_(def)
317  , default_defines_()
319  , location_("")
320  , linenum_(0)
321  , quoted_(false)
322  {
323  }
324 
325  /** Decodes the filenames placed in a location. */
326  std::string get_current_file();
327 
328  void error(const std::string&, int);
329  void warning(const std::string&, int);
330 
331  template<typename T, typename... A>
332  void add_preprocessor(A&&... args)
333  {
334  preprocessor_queue_.emplace_back(new T(*this, std::forward<A>(args)...));
335  preprocessor_queue_.back()->init();
336  }
337 
339  {
340  preprocessor_queue_.pop_back();
341  }
342 
343  int depth() const
344  {
345  return preprocessor_queue_.size();
346  }
347 
349  {
350  return preprocessor_queue_.empty() ? nullptr : preprocessor_queue_.back().get();
351  }
352 
353 private:
355  : std::streambuf()
356  , out_buffer_("")
357  , buffer_()
359  , defines_(t.defines_)
360  , default_defines_()
362  , location_("")
363  , linenum_(0)
364  , quoted_(t.quoted_)
365  {
366  }
367 
368  /** Inherited from basic_streambuf. */
369  virtual int underflow() override;
370 
372 
373  /** Buffer read by the STL stream. */
374  std::string out_buffer_;
375 
376  /** Buffer filled by the _current_ preprocessor. */
377  std::stringstream buffer_;
378 
379  /** Input preprocessor queue. */
380  std::deque<std::unique_ptr<preprocessor>> preprocessor_queue_;
381 
384 
385  std::string textdomain_;
386  std::string location_;
387 
388  int linenum_;
389 
390  /**
391  * Set to true if one preprocessor for this target started to read a string.
392  * Deeper-nested preprocessors are then forbidden to.
393  */
394  bool quoted_;
395 
396  friend class preprocessor;
397  friend class preprocessor_file;
398  friend class preprocessor_data;
400 };
401 
402 /** Preprocessor constructor. */
404  : parent_(t)
405  , old_textdomain_(t.textdomain_)
406  , old_location_(t.location_)
407  , old_linenum_(t.linenum_)
408 {
409 }
410 
411 /**
412  * Called by an STL stream whenever it has reached the end of #out_buffer_.
413  * Fills #buffer_ by calling the _current_ preprocessor, then copies its
414  * content into #out_buffer_.
415  * @return the first character of #out_buffer_ if any, EOF otherwise.
416  */
418 {
419  unsigned sz = 0;
420  if(char* gp = gptr()) {
421  if(gp < egptr()) {
422  // Sanity check: the internal buffer has not been totally consumed,
423  // should we force the caller to use what remains first?
424  return *gp;
425  }
426 
427  // The buffer has been completely read; fill it again.
428  // Keep part of the previous buffer, to ensure putback capabilities.
429  sz = out_buffer_.size();
430  buffer_.str(std::string());
431 
432  if(sz > 3) {
433  buffer_ << out_buffer_.substr(sz - 3);
434  sz = 3;
435  } else {
436  buffer_ << out_buffer_;
437  }
438  } else {
439  // The internal get-data pointer is null
440  }
441 
442  const int desired_fill_amount = 2000;
443 
444  while(current() && buffer_.rdbuf()->in_avail() < desired_fill_amount) {
445  // Process files and data chunks until the desired buffer size is reached
446  if(!current()->get_chunk()) {
447  // Drop the current preprocessor item from the queue.
449  }
450  }
451 
452  // Update the internal state and data pointers
453  out_buffer_ = buffer_.str();
454  if(out_buffer_.empty()) {
455  return EOF;
456  }
457 
458  char* begin = &*out_buffer_.begin();
459  unsigned bs = out_buffer_.size();
460 
461  setg(begin, begin + sz, begin + bs);
462 
463  if(sz >= bs) {
464  return EOF;
465  }
466 
467  return static_cast<unsigned char>(*(begin + sz));
468 }
469 
470 /**
471 * Restores the old preprocessing context.
472 * Appends location and domain directives to the buffer, so that the parser
473 * notices these changes.
474 */
476 {
477  preprocessor* current = this->current();
478 
479  if(!current->old_location_.empty()) {
481  }
482 
484  buffer_ << INLINED_PREPROCESS_DIRECTIVE_CHAR << "textdomain " << current->old_textdomain_ << '\n';
485  }
486 
490 
491  // Drop the preprocessor from the queue.
493 }
494 
496 {
497  unsigned nested_level = 0;
498 
499  preprocessor* pre = nullptr;
500 
501  // Iterate backwards over queue to get the last non-macro preprocessor.
502  for(auto p = preprocessor_queue_.rbegin(); p != preprocessor_queue_.rend(); ++p) {
503  pre = p->get();
504 
505  if(!pre || pre->parse_mode() == preprocessor::PARSES_FILE) {
506  break;
507  }
508 
509  if(pre->parse_mode() == preprocessor::PARSES_MACRO) {
510  ++nested_level;
511  }
512  }
513 
514  std::string res;
515  std::vector<std::string> pos = utils::quoted_split(location_, ' ');
516 
517  if(pos.size() <= 2 * nested_level) {
518  return res;
519  }
520 
521  return get_filename(pos[2 * nested_level]);
522 }
523 
524 std::string lineno_string(const std::string& lineno)
525 {
526  std::vector<std::string> pos = utils::quoted_split(lineno, ' ');
527  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
528  std::string included_from = preprocessor_error_detail_prefix + "included from ";
529  std::string res;
530 
531  while(i != end) {
532  const std::string& line = *(i++);
533 
534  if(!res.empty()) {
535  res += included_from;
536  }
537 
538  if(i != end) {
539  res += get_filename(*(i++));
540  } else {
541  res += "<unknown>";
542  }
543 
544  res += ':' + line;
545  }
546 
547  if(res.empty()) {
548  res = "???";
549  }
550 
551  return res;
552 }
553 
554 void preprocessor_streambuf::error(const std::string& error_type, int l)
555 {
556  std::string position, error;
557  std::ostringstream pos;
558 
559  pos << l << ' ' << location_;
560  position = lineno_string(pos.str());
561 
562  error = error_type + '\n';
563  error += "at " + position;
564 
565  ERR_PREPROC << error;
566 
568 }
569 
570 void preprocessor_streambuf::warning(const std::string& warning_type, int l)
571 {
572  std::string position, warning;
573  std::ostringstream pos;
574 
575  pos << l << ' ' << location_;
576  position = lineno_string(pos.str());
577 
578  warning = warning_type + '\n';
579  warning += "at " + position;
580 
581  WRN_PREPROC << warning;
582 }
583 
584 
585 // ==================================================================================
586 // PREPROCESSOR FILE
587 // ==================================================================================
588 
589 /**
590  * Specialized preprocessor for handling a file or a set of files.
591  * A preprocessor_file object is created when a preprocessor encounters an
592  * inclusion directive that resolves to a file or directory, e.g. '{themes/}'.
593  */
595 {
596 public:
597  /** Constructor. It relies on preprocessor_data so it's implemented after that class is declared. */
598  preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index = -1);
599 
600  virtual void init() override;
601 
602  /**
603  * Inserts and processes the next file in the list of included files.
604  * @return false if there is no next file.
605  */
606  virtual bool get_chunk() override
607  {
608  while(pos_ != end_) {
609  const std::string& name = *(pos_++);
610  unsigned sz = name.size();
611 
612  // Use reverse iterator to optimize testing
613  if(sz < 5 || !std::equal(name.rbegin(), name.rbegin() + 4, "gfc.")) {
614  continue;
615  }
616 
618  return true;
619  }
620 
621  return false;
622  }
623 
624 private:
625  std::vector<std::string> files_;
626  std::vector<std::string>::const_iterator pos_, end_;
627 
628  const std::string& name_;
629 
631 };
632 
633 
634 // ==================================================================================
635 // PREPROCESSOR DATA
636 // ==================================================================================
637 
638 /**
639  * Specialized preprocessor for handling any kind of input stream.
640  * This is the core of the preprocessor.
641  */
643 {
644  /** Description of a preprocessing chunk. */
645  struct token_desc
646  {
647  enum class token_type {
648  start, // Toplevel
649  process_if, // Processing the "if" branch of a ifdef/ifndef (the "else" branch will be skipped)
650  process_else, // Processing the "else" branch of a ifdef/ifndef
651  skip_if, // Skipping the "if" branch of a ifdef/ifndef (the "else" branch, if any, will be processed)
652  skip_else, // Skipping the "else" branch of a ifdef/ifndef
653  string, // Processing a string
654  verbatim, // Processing a verbatim string
655  macro_space, // Processing between chunks of a macro call (skip spaces)
656  macro_chunk, // Processing inside a chunk of a macro call (stop on space or '(')
657  macro_parens // Processing a parenthesized macro argument
658  };
659 
660  token_desc(token_type type, const int stack_pos, const int linenum)
661  : type(type)
663  , linenum(linenum)
664  {
665  }
666 
668 
669  /** Starting position in #strings_ of the delayed text for this chunk. */
671  int linenum;
672  };
673 
674  /**
675  * Manages the lifetime of the @c std::istream pointer we own.
676  *
677  * Since @ref in_ uses the stream as well this object must be created
678  * before @ref in_ and destroyed after @ref in_ is destroyed.
679  */
681 
682  /** Input stream. */
684 
685  std::string directory_;
686 
687  /** Buffer for delayed input processing. */
688  std::vector<std::string> strings_;
689 
690  /** Mapping of macro arguments to their content. */
691  std::unique_ptr<std::map<std::string, std::string>> local_defines_;
692 
693  /** Stack of nested preprocessing chunks. */
694  std::vector<token_desc> tokens_;
695 
696  /**
697  * Set to true whenever input tokens cannot be directly sent to the target
698  * buffer. For instance, this happens with macro arguments. In that case,
699  * the output is redirected toward #strings_ until it can be processed.
700  */
702 
703  /**
704  * Non-zero when the preprocessor has to skip some input text.
705  * Increased whenever entering a conditional branch that is not useful,
706  * e.g. a ifdef that evaluates to false.
707  */
709  int linenum_;
710 
711  /** True iff we are currently parsing a macros content, otherwise false. */
713 
714  std::string read_word();
715  std::string read_line();
716  std::string read_rest_of_line();
717 
718  void skip_spaces();
719  void skip_eol();
721  void pop_token();
722  void put(char);
723  void put(const std::string& /*, int change_line = 0 */);
724  void conditional_skip(bool skip);
725 
726 public:
729  const std::string& history,
730  const std::string& name,
731  int line,
732  const std::string& dir,
733  const std::string& domain,
734  std::unique_ptr<std::map<std::string, std::string>> defines,
735  bool is_define = false);
736 
737  virtual bool get_chunk() override;
738 
739  virtual preprocessor::MODE parse_mode() override
740  {
742  }
743 
748 };
749 
751 {
752  throw std::logic_error("don't compare tokens with characters");
753 }
754 
756 {
757  return rhs == lhs;
758 }
759 
761 {
762  return !(lhs == rhs);
763 }
764 
766 {
767  return rhs != lhs;
768 }
769 
770 /** preprocessor_file constructor. */
771 preprocessor_file::preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index)
772  : preprocessor(t)
773  , files_()
774  , pos_()
775  , end_()
776  , name_(name)
777  , is_directory_(filesystem::is_directory(name))
778 {
779  if(is_directory_) {
780  filesystem::get_files_in_dir(name, &files_, nullptr,
784  );
785 
786  for(const std::string& fname : files_) {
787  std::size_t cpos = fname.rfind(" ");
788 
789  if(cpos != std::string::npos && cpos >= symbol_index) {
790  std::stringstream ss;
791  ss << "Found filename containing whitespace: '" << filesystem::base_name(fname)
792  << "' in included directory '" << name << "'.\nThe included symbol probably looks similar to '"
793  << filesystem::directory_name(fname.substr(symbol_index)) << "'";
794 
795  // TODO: find a real linenumber
796  parent_.error(ss.str(), -1);
797  }
798  }
799  } else {
800  // Handled in the init() function.
801  }
802 
803  pos_ = files_.begin();
804  end_ = files_.end();
805 }
806 
808 {
809  if(is_directory_) {
810  return;
811  }
812 
814 
815  if(!file_stream->good()) {
816  ERR_PREPROC << "Could not open file " << name_;
817  } else {
818  parent_.add_preprocessor<preprocessor_data>(std::move(file_stream), "",
821  );
822  }
823 }
824 
827  const std::string& history,
828  const std::string& name,
829  int linenum,
830  const std::string& directory,
831  const std::string& domain,
832  std::unique_ptr<std::map<std::string, std::string>> defines,
833  bool is_define)
834  : preprocessor(t)
835  , in_scope_(std::move(i))
836  , in_(*in_scope_)
837  , directory_(directory)
838  , strings_()
839  , local_defines_(std::move(defines))
840  , tokens_()
841  , slowpath_(0)
842  , skipping_(0)
843  , linenum_(linenum)
844  , is_define_(is_define)
845 {
846  std::ostringstream s;
847  s << history;
848 
849  if(!name.empty()) {
850  if(!history.empty()) {
851  s << ' ';
852  }
853 
854  s << get_file_code(name);
855  }
856 
857  if(!t.location_.empty()) {
858  s << ' ' << t.linenum_ << ' ' << t.location_;
859  }
860 
861  t.location_ = s.str();
862  t.linenum_ = linenum;
863 
864  t.buffer_ << INLINED_PREPROCESS_DIRECTIVE_CHAR << "line " << linenum << ' ' << t.location_ << '\n';
865 
866  if(t.textdomain_ != domain) {
867  t.buffer_ << INLINED_PREPROCESS_DIRECTIVE_CHAR << "textdomain " << domain << '\n';
868  t.textdomain_ = domain;
869  }
870 
872 }
873 
875 {
876  tokens_.emplace_back(t, strings_.size(), linenum_);
877 
879  // Macro expansions do not have any associated storage at start.
880  return;
882  /* Quoted strings are always inlined in the parent token. So
883  * they need neither storage nor metadata, unless the parent
884  * token is a macro expansion.
885  */
886  token_desc::token_type& outer_type = tokens_[tokens_.size() - 2].type;
887  if(outer_type != token_desc::token_type::macro_space) {
888  return;
889  }
890 
892  tokens_.back().stack_pos = strings_.size() + 1;
893  }
894 
895  std::ostringstream s;
896  if(!skipping_ && slowpath_) {
897  s << INLINED_PREPROCESS_DIRECTIVE_CHAR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
898  << INLINED_PREPROCESS_DIRECTIVE_CHAR << "textdomain " << parent_.textdomain_ << '\n';
899  }
900 
901  strings_.push_back(s.str());
902 }
903 
905 {
906  token_desc::token_type inner_type = tokens_.back().type;
907  unsigned stack_pos = tokens_.back().stack_pos;
908 
909  tokens_.pop_back();
910 
911  token_desc::token_type& outer_type = tokens_.back().type;
912 
913  if(inner_type == token_desc::token_type::macro_parens) {
914  // Parenthesized macro arguments are left on the stack.
915  assert(outer_type == token_desc::token_type::macro_space);
916  return;
917  }
918 
919  if(inner_type == token_desc::token_type::string || inner_type == token_desc::token_type::verbatim) {
920  // Quoted strings are always inlined.
921  assert(stack_pos == strings_.size());
922  return;
923  }
924 
925  if(outer_type == token_desc::token_type::macro_space) {
926  /* A macro expansion does not have any associated storage.
927  * Instead, storage of the inner token is not discarded
928  * but kept as a new macro argument. But if the inner token
929  * was a macro expansion, it is about to be appended, so
930  * prepare for it.
931  */
933  strings_.erase(strings_.begin() + stack_pos, strings_.end());
934  strings_.emplace_back();
935  }
936 
937  assert(stack_pos + 1 == strings_.size());
939 
940  return;
941  }
942 
943  strings_.erase(strings_.begin() + stack_pos, strings_.end());
944 }
945 
947 {
948  while(true) {
949  int c = in_.peek();
950 
951  if(in_.eof() || (c != ' ' && c != '\t')) {
952  return;
953  }
954 
955  in_.get();
956  }
957 }
958 
960 {
961  while(true) {
962  int c = in_.get();
963 
964  if(c == '\n') {
965  ++linenum_;
966  return;
967  }
968 
969  if(in_.eof()) {
970  return;
971  }
972  }
973 }
974 
976 {
977  std::string res;
978 
979  while(true) {
980  int c = in_.peek();
981 
982  if(c == preprocessor_streambuf::traits_type::eof() || utils::portable_isspace(c)) {
983  // DBG_PREPROC << "(" << res << ")";
984  return res;
985  }
986 
987  in_.get();
988  res += static_cast<char>(c);
989  }
990 }
991 
993 {
994  std::string res;
995 
996  while(true) {
997  int c = in_.get();
998 
999  if(c == '\n') {
1000  ++linenum_;
1001  return res;
1002  }
1003 
1004  if(in_.eof()) {
1005  return res;
1006  }
1007 
1008  if(c != '\r') {
1009  res += static_cast<char>(c);
1010  }
1011  }
1012 }
1013 
1015 {
1016  std::string res;
1017 
1018  while(in_.peek() != '\n' && !in_.eof()) {
1019  int c = in_.get();
1020 
1021  if(c != '\r') {
1022  res += static_cast<char>(c);
1023  }
1024  }
1025 
1026  return res;
1027 }
1028 
1030 {
1031  if(skipping_) {
1032  return;
1033  }
1034 
1035  if(slowpath_) {
1036  strings_.back() += c;
1037  return;
1038  }
1039 
1040  int cond_linenum = c == '\n' ? linenum_ - 1 : linenum_;
1041 
1042  if(unsigned diff = cond_linenum - parent_.linenum_) {
1043  parent_.linenum_ = cond_linenum;
1044 
1045  if(diff <= parent_.location_.size() + 11) {
1046  parent_.buffer_ << std::string(diff, '\n');
1047  } else {
1049  }
1050  }
1051 
1052  if(c == '\n') {
1053  ++parent_.linenum_;
1054  }
1055 
1056  parent_.buffer_ << c;
1057 }
1058 
1059 void preprocessor_data::put(const std::string& s /*, int line_change*/)
1060 {
1061  if(skipping_) {
1062  return;
1063  }
1064 
1065  if(slowpath_) {
1066  strings_.back() += s;
1067  return;
1068  }
1069 
1070  parent_.buffer_ << s;
1071  // parent_.linenum_ += line_change;
1072 }
1073 
1075 {
1076  if(skip) {
1077  ++skipping_;
1078  }
1079 
1081 }
1082 
1084 {
1085  char c = static_cast<char>(in_.get());
1086  token_desc& token = tokens_.back();
1087 
1088  if(in_.eof()) {
1089  // The end of file was reached.
1090  // Make sure we don't have any incomplete tokens.
1091  char const* s;
1092 
1093  switch(token.type) {
1095  return false; // everything is fine
1100  s = "#ifdef or #ifndef";
1101  break;
1103  s = "Quoted string";
1104  break;
1106  s = "Verbatim string";
1107  break;
1110  s = "Macro substitution";
1111  break;
1113  s = "Macro argument";
1114  break;
1115  default:
1116  s = "???";
1117  }
1118 
1119  parent_.error(std::string(s) + " not terminated", token.linenum);
1120  }
1121 
1122  if(c == '\n') {
1123  ++linenum_;
1124  }
1125 
1126  if(c == static_cast<char>(INLINED_PREPROCESS_DIRECTIVE_CHAR)) {
1127  std::string buffer(1, c);
1128 
1129  while(true) {
1130  char d = static_cast<char>(in_.get());
1131 
1132  if(in_.eof() || d == '\n') {
1133  break;
1134  }
1135 
1136  buffer += d;
1137  }
1138 
1139  buffer += '\n';
1140  // line_change = 1-1 = 0
1141  put(buffer);
1143  put(c);
1144 
1145  if(c == '>' && in_.peek() == '>') {
1146  put(in_.get());
1147  pop_token();
1148  }
1149  } else if(c == '<' && in_.peek() == '<') {
1150  in_.get();
1152  put('<');
1153  put('<');
1154  } else if(c == '"') {
1156  parent_.quoted_ = false;
1157  put(c);
1158  pop_token();
1159  } else if(!parent_.quoted_) {
1160  parent_.quoted_ = true;
1162  put(c);
1163  } else {
1164  parent_.error("Nested quoted string", linenum_);
1165  }
1166  } else if(c == '{') {
1168  ++slowpath_;
1169  } else if(c == ')' && token.type == token_desc::token_type::macro_parens) {
1170  pop_token();
1171  } else if(c == '#' && !parent_.quoted_) {
1172  std::string command = read_word();
1173  bool comment = false;
1174 
1175  if(command == "define") {
1176  skip_spaces();
1177  int linenum = linenum_;
1178  std::vector<std::string> items = utils::split(read_line(), ' ');
1179  std::map<std::string, std::string> optargs;
1180 
1181  if(items.empty()) {
1182  parent_.error("No macro name found after #define directive", linenum);
1183  }
1184 
1185  std::string symbol = items.front();
1186  items.erase(items.begin());
1187  int found_arg = 0, found_enddef = 0, found_deprecate = 0;
1188  utils::optional<DEP_LEVEL> deprecation_level;
1189  std::string buffer, deprecation_detail;
1190  version_info deprecation_version = game_config::wesnoth_version;
1191  while(true) {
1192  if(in_.eof())
1193  break;
1194  char d = static_cast<char>(in_.get());
1195  if(d == '\n')
1196  ++linenum_;
1197  buffer += d;
1198  if(d == '#') {
1199  if(in_.peek() == 'a') {
1200  found_arg = 1;
1201  } else if(in_.peek() == 'd') {
1202  found_deprecate = 1;
1203  } else {
1204  found_enddef = 1;
1205  }
1206  } else {
1207  if(found_arg > 0 && ++found_arg == 4) {
1208  if(std::equal(buffer.end() - 3, buffer.end(), "arg")) {
1209  buffer.erase(buffer.end() - 4, buffer.end());
1210 
1211  skip_spaces();
1212  std::string argname = read_word();
1213  skip_eol();
1214 
1215  std::string argbuffer;
1216 
1217  int found_endarg = 0;
1218  while(true) {
1219  if(in_.eof()) {
1220  break;
1221  }
1222 
1223  char e = static_cast<char>(in_.get());
1224  if(e == '\n') {
1225  ++linenum_;
1226  }
1227 
1228  argbuffer += e;
1229 
1230  if(e == '#') {
1231  found_endarg = 1;
1232  } else if(found_endarg > 0 && ++found_endarg == 7) {
1233  if(std::equal(argbuffer.end() - 6, argbuffer.end(), "endarg")) {
1234  argbuffer.erase(argbuffer.end() - 7, argbuffer.end());
1235  optargs[argname] = argbuffer;
1236  skip_eol();
1237  break;
1238  } else {
1239  parent_.error("Unterminated #arg definition", linenum_);
1240  }
1241  }
1242  }
1243  }
1244  }
1245 
1246  if(found_deprecate > 0 && ++found_deprecate == 11) {
1247  if(std::equal(buffer.end() - 10, buffer.end(), "deprecated")) {
1248  buffer.erase(buffer.end() - 11, buffer.end());
1249  skip_spaces();
1250  try {
1252  if(deprecation_level) {
1253  deprecation_level = std::max(*deprecation_level, level);
1254  } else {
1255  deprecation_level = level;
1256  }
1257  } catch(const std::invalid_argument&) {
1258  // Meh, fall back to default of PREEMPTIVE...
1259  deprecation_level = DEP_LEVEL::PREEMPTIVE;
1260  }
1261  deprecation_version = game_config::wesnoth_version;
1262  if(deprecation_level == DEP_LEVEL::PREEMPTIVE || deprecation_level == DEP_LEVEL::FOR_REMOVAL) {
1263  skip_spaces();
1264  deprecation_version = std::max(deprecation_version, version_info(read_word()));
1265  }
1266  skip_spaces();
1267  if(!deprecation_detail.empty()){
1268  deprecation_detail += '\n';
1269  }
1270  deprecation_detail += read_rest_of_line();
1271  skip_eol();
1272  }
1273  }
1274 
1275  if(found_enddef > 0 && ++found_enddef == 7) {
1276  if(std::equal(buffer.end() - 6, buffer.end(), "enddef")) {
1277  break;
1278  } else {
1279  found_enddef = 0;
1280  if(std::equal(buffer.end() - 6, buffer.end(), "define")) { // TODO: Maybe add support for
1281  // this? This would fill feature
1282  // request #21343
1283  parent_.error(
1284  "Preprocessor error: #define is not allowed inside a #define/#enddef pair",
1285  linenum);
1286  }
1287  }
1288  }
1289  }
1290  }
1291 
1292  if(found_enddef != 7) {
1293  parent_.error("Unterminated preprocessor definition", linenum_);
1294  }
1295 
1296  if(!skipping_) {
1297  preproc_map::const_iterator old_i = parent_.defines_->find(symbol);
1298  if(old_i != parent_.defines_->end()) {
1299  std::ostringstream new_pos, old_pos;
1300  const preproc_define& old_d = old_i->second;
1301 
1302  new_pos << linenum << ' ' << parent_.location_;
1303  old_pos << old_d.linenum << ' ' << old_d.location;
1304 
1305  WRN_PREPROC << "Redefining macro " << symbol << " without explicit #undef at "
1306  << lineno_string(new_pos.str()) << '\n'
1307  << "previously defined at " << lineno_string(old_pos.str());
1308  }
1309 
1310  buffer.erase(buffer.end() - 7, buffer.end());
1311  (*parent_.defines_).insert_or_assign(symbol,
1312  preproc_define(buffer, items, optargs, parent_.textdomain_, linenum, parent_.location_,
1313  deprecation_detail, deprecation_level, deprecation_version));
1314 
1315  LOG_PREPROC << "defining macro " << symbol << " (location " << get_location(parent_.location_) << ")";
1316  }
1317  } else if(command == "ifdef" || command == "ifndef") {
1318  const bool negate = command[2] == 'n';
1319  skip_spaces();
1320  const std::string& symbol = read_word();
1321  if(symbol.empty()) {
1322  parent_.error("No macro argument found after #ifdef/#ifndef directive", linenum_);
1323  }
1324  bool found = parent_.defines_->count(symbol) != 0;
1325  DBG_PREPROC << "testing for macro " << symbol << ": " << (found ? "defined" : "not defined");
1326  conditional_skip(negate ? found : !found);
1327  } else if(command == "ifhave" || command == "ifnhave") {
1328  const bool negate = command[2] == 'n';
1329  skip_spaces();
1330  const std::string& symbol = read_word();
1331  if(symbol.empty()) {
1332  parent_.error("No path argument found after #ifhave/#ifnhave directive", linenum_);
1333  }
1334  bool found = filesystem::get_wml_location(symbol, directory_).has_value();
1335  DBG_PREPROC << "testing for file or directory " << symbol << ": " << (found ? "found" : "not found");
1336  conditional_skip(negate ? found : !found);
1337  } else if(command == "ifver" || command == "ifnver") {
1338  const bool negate = command[2] == 'n';
1339 
1340  skip_spaces();
1341  const std::string& vsymstr = read_word();
1342  skip_spaces();
1343  const std::string& vopstr = read_word();
1344  skip_spaces();
1345  const std::string& vverstr = read_word();
1346 
1347  const VERSION_COMP_OP vop = parse_version_op(vopstr);
1348 
1349  if(vop == OP_INVALID) {
1350  parent_.error("Invalid #ifver/#ifnver operator", linenum_);
1351  } else if(parent_.defines_->count(vsymstr) != 0) {
1352  const preproc_define& sym = (*parent_.defines_)[vsymstr];
1353 
1354  if(!sym.arguments.empty()) {
1355  parent_.error("First argument macro in #ifver/#ifnver should not require arguments", linenum_);
1356  }
1357 
1358  version_info const version1(sym.value);
1359  version_info const version2(vverstr);
1360 
1361  const bool found = do_version_check(version1, vop, version2);
1362  DBG_PREPROC << "testing version '" << version1.str() << "' against '" << version2.str() << "' ("
1363  << vopstr << "): " << (found ? "match" : "no match");
1364 
1365  conditional_skip(negate ? found : !found);
1366  } else {
1367  std::string err = "Undefined macro in #ifver/#ifnver first argument: '";
1368  err += vsymstr;
1369  err += "'";
1371  }
1372  } else if(command == "else") {
1374  pop_token();
1375  --skipping_;
1378  pop_token();
1379  ++skipping_;
1381  } else {
1382  parent_.error("Unexpected #else", linenum_);
1383  }
1384  } else if(command == "endif") {
1385  switch(token.type) {
1388  --skipping_;
1391  break;
1392  default:
1393  parent_.error("Unexpected #endif", linenum_);
1394  }
1395  pop_token();
1396  } else if(command == "textdomain") {
1397  skip_spaces();
1398  const std::string& s = read_word();
1399  if(s != parent_.textdomain_) {
1400  put("#textdomain ");
1401  put(s);
1402  parent_.textdomain_ = s;
1403  }
1404  comment = true;
1405  } else if(command == "enddef") {
1406  parent_.error("Unexpected #enddef", linenum_);
1407  } else if(command == "undef") {
1408  skip_spaces();
1409  const std::string& symbol = read_word();
1410  if(!skipping_) {
1411  parent_.defines_->erase(symbol);
1412  LOG_PREPROC << "undefine macro " << symbol << " (location " << get_location(parent_.location_) << ")";
1413  }
1414  } else if(command == "error") {
1415  if(!skipping_) {
1416  skip_spaces();
1417  std::ostringstream error;
1418  error << "#error: \"" << read_rest_of_line() << '"';
1419  parent_.error(error.str(), linenum_);
1420  } else
1421  DBG_PREPROC << "Skipped an error";
1422  } else if(command == "warning") {
1423  if(!skipping_) {
1424  skip_spaces();
1425  std::ostringstream warning;
1426  warning << "#warning: \"" << read_rest_of_line() << '"';
1427  parent_.warning(warning.str(), linenum_);
1428  } else {
1429  DBG_PREPROC << "Skipped a warning";
1430  }
1431  } else if(command == "deprecated") {
1432  // The current file is deprecated, so print a message
1433  skip_spaces();
1435  try {
1437  } catch(const std::invalid_argument&) {
1438  // Meh, just fall back to the default of PREEMPTIVE...
1439  }
1442  skip_spaces();
1443  version = version_info(read_word());
1444  }
1445  skip_spaces();
1446  std::string detail = read_rest_of_line();
1448  } else {
1450  }
1451 
1452  skip_eol();
1453  if(comment) {
1454  put('\n');
1455  }
1457  if(c == '(') {
1458  // If a macro argument was started, it is implicitly ended.
1461  } else if(utils::portable_isspace(c)) {
1462  // If a macro argument was started, it is implicitly ended.
1464  } else if(c == '}') {
1465  --slowpath_;
1466  if(skipping_) {
1467  pop_token();
1468  return true;
1469  }
1470 
1471  // FIXME: is this obsolete?
1472  // if (token.type == token_desc::MACRO_SPACE) {
1473  // if (!strings_.back().empty()) {
1474  // std::ostringstream error;
1475  // std::ostringstream location;
1476  // error << "Can't parse new macro parameter with a macro call scope open";
1477  // location<<linenum_<<' '<<parent_.location_;
1478  // parent_.error(error.str(), location.str());
1479  // }
1480  // strings_.pop_back();
1481  //}
1482 
1483  if(strings_.size() <= static_cast<std::size_t>(token.stack_pos)) {
1484  parent_.error("No macro or file substitution target specified", linenum_);
1485  }
1486 
1487  std::string symbol = strings_[token.stack_pos];
1488  std::string::size_type pos;
1489  while((pos = symbol.find(INLINED_PREPROCESS_DIRECTIVE_CHAR)) != std::string::npos) {
1490  std::string::iterator b = symbol.begin(); // invalidated at each iteration
1491  symbol.erase(b + pos, b + symbol.find('\n', pos + 1) + 1);
1492  }
1493 
1494  std::map<std::string, std::string>::const_iterator arg;
1495  preproc_map::const_iterator macro;
1496 
1497  // If this is a known pre-processing symbol, then we insert it,
1498  // otherwise we assume it's a file name to load.
1499  if(symbol == current_file_str && strings_.size() - token.stack_pos == 1) {
1500  pop_token();
1502  } else if(symbol == current_dir_str && strings_.size() - token.stack_pos == 1) {
1503  pop_token();
1505  } else if(symbol == left_curly_str && strings_.size() - token.stack_pos == 1) {
1506  pop_token();
1507  put("{");
1508  } else if(symbol == right_curly_str && strings_.size() - token.stack_pos == 1) {
1509  pop_token();
1510  put("}");
1511  } else if(local_defines_ && (arg = local_defines_->find(symbol)) != local_defines_->end()) {
1512  if(strings_.size() - token.stack_pos != 1) {
1513  std::ostringstream error;
1514  error << "Macro argument '" << symbol << "' does not expect any arguments";
1515  parent_.error(error.str(), linenum_);
1516  }
1517 
1518  std::ostringstream v;
1519  v << arg->second << INLINED_PREPROCESS_DIRECTIVE_CHAR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1520  << INLINED_PREPROCESS_DIRECTIVE_CHAR << "textdomain " << parent_.textdomain_ << '\n';
1521 
1522  pop_token();
1523  put(v.str());
1524  } else if(parent_.depth() < 100 && (macro = parent_.defines_->find(symbol)) != parent_.defines_->end()) {
1525  const preproc_define& val = macro->second;
1526  std::size_t nb_arg = strings_.size() - token.stack_pos - 1;
1527  std::size_t optional_arg_num = 0;
1528 
1529  auto defines = std::make_unique<std::map<std::string, std::string>>();
1530  const std::string& dir = filesystem::directory_name(val.location.substr(0, val.location.find(' ')));
1531 
1532  if(val.is_deprecated()) {
1534  }
1535 
1536  for(std::size_t i = 0; i < nb_arg; ++i) {
1537  if(i < val.arguments.size()) {
1538  // Normal mandatory arguments
1539 
1540  (*defines)[val.arguments[i]] = strings_[token.stack_pos + i + 1];
1541  } else {
1542  // These should be optional argument overrides
1543 
1544  std::string str = strings_[token.stack_pos + i + 1];
1545  std::size_t equals_pos = str.find_first_of("=");
1546 
1547  if(equals_pos != std::string::npos) {
1548  std::size_t argname_pos = str.substr(0, equals_pos).find_last_of(" \n") + 1;
1549 
1550  std::string argname = str.substr(argname_pos, equals_pos - argname_pos);
1551 
1552  if(val.optional_arguments.find(argname) != val.optional_arguments.end()) {
1553  (*defines)[argname] = str.substr(equals_pos + 1);
1554 
1555  optional_arg_num++;
1556 
1557  DBG_PREPROC << "Found override for " << argname << " in call to macro " << symbol;
1558  } else {
1559  std::ostringstream warning;
1560  warning << "Unrecognized optional argument passed to macro '" << symbol << "': '"
1561  << argname << "'";
1562  parent_.warning(warning.str(), linenum_);
1563 
1564  optional_arg_num++; // To prevent the argument number check from blowing up
1565  }
1566  }
1567  }
1568  }
1569 
1570  // If the macro definition has any optional arguments, insert their defaults
1571  if(val.optional_arguments.size() > 0) {
1572  for(const auto& argument : val.optional_arguments) {
1573  if(defines->find(argument.first) == defines->end()) {
1574  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1575 
1576  buf->textdomain_ = parent_.textdomain_;
1577  std::istream in(buf.get());
1578 
1579  filesystem::scoped_istream buffer{new std::istringstream(argument.second)};
1580 
1581  auto temp_defines = std::make_unique<std::map<std::string, std::string>>();
1582  temp_defines->insert(defines->begin(), defines->end());
1583 
1584  buf->add_preprocessor<preprocessor_data>(
1585  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(temp_defines), false);
1586 
1587  std::ostringstream res;
1588  res << in.rdbuf();
1589 
1590  DBG_PREPROC << "Setting default for optional argument " << argument.first << " in macro "
1591  << symbol;
1592 
1593  (*defines)[argument.first] = res.str();
1594  }
1595  }
1596  }
1597 
1598  if(nb_arg - optional_arg_num != val.arguments.size()) {
1599  const std::vector<std::string>& locations = utils::quoted_split(val.location, ' ');
1600  const std::string filename = locations.empty() ? "<command-line>" : get_filename(locations[0]);
1601  std::ostringstream error;
1602  error << "Preprocessor symbol '" << symbol << "' defined at " << filename << ":"
1603  << val.linenum << " expects " << val.arguments.size() << " arguments, but has "
1604  << nb_arg - optional_arg_num << " arguments";
1605  parent_.error(error.str(), linenum_);
1606  }
1607 
1608  filesystem::scoped_istream buffer{new std::istringstream(val.value)};
1609 
1610  pop_token();
1611 
1612  if(!slowpath_) {
1613  DBG_PREPROC << "substituting macro " << symbol;
1614 
1616  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1617  } else {
1618  DBG_PREPROC << "substituting (slow) macro " << symbol;
1619 
1620  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1621 
1622  // Make the nested preprocessor_data responsible for
1623  // restoring our current textdomain if needed.
1624  buf->textdomain_ = parent_.textdomain_;
1625 
1626  std::ostringstream res;
1627  {
1628  std::istream in(buf.get());
1629  buf->add_preprocessor<preprocessor_data>(
1630  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1631 
1632  res << in.rdbuf();
1633  }
1634 
1635  put(res.str());
1636  }
1637  } else if(parent_.depth() < 40) {
1638  LOG_PREPROC << "Macro definition not found for " << symbol << ", attempting to open as file.";
1639  pop_token();
1640 
1641  if(auto nfname = filesystem::get_wml_location(symbol, directory_)) {
1642  if(!slowpath_)
1643  // nfname.size() - symbol.size() gives you an index into nfname
1644  // This does not necessarily match the symbol though, as it can start with ~ or ./
1645  parent_.add_preprocessor<preprocessor_file>(nfname.value(), nfname->size() - symbol.size());
1646  else {
1647  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1648 
1649  std::ostringstream res;
1650  {
1651  std::istream in(buf.get());
1652  buf->add_preprocessor<preprocessor_file>(nfname.value(), nfname->size() - symbol.size());
1653 
1654  res << in.rdbuf();
1655  }
1656 
1657  put(res.str());
1658  }
1659  } else {
1660  std::ostringstream error;
1661  error << "Macro/file '" << symbol << "' is missing";
1662  parent_.error(error.str(), linenum_);
1663  }
1664  } else {
1665  parent_.error("Too many nested preprocessing inclusions", linenum_);
1666  }
1667  } else if(!skipping_) {
1669  std::ostringstream s;
1670  s << INLINED_PREPROCESS_DIRECTIVE_CHAR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1671  << INLINED_PREPROCESS_DIRECTIVE_CHAR << "textdomain " << parent_.textdomain_ << '\n';
1672 
1673  strings_.push_back(s.str());
1675  }
1676  put(c);
1677  }
1678  } else {
1679  put(c);
1680  }
1681 
1682  return true;
1683 }
1684 
1685 
1686 // ==================================================================================
1687 // PREPROCESSOR SCOPE HELPER
1688 // ==================================================================================
1689 
1690 struct preprocessor_scope_helper : std::basic_istream<char>
1691 {
1692  preprocessor_scope_helper(const std::string& fname, preproc_map* defines)
1693  : std::basic_istream<char>(nullptr)
1694  , buf_(nullptr)
1695  , local_defines_(nullptr)
1696  {
1697  //
1698  // If no defines were provided, we create a new local preproc_map and assign
1699  // it to defines temporarily. In this case, the map will be deleted once this
1700  // object is destroyed and defines will still be subsequently null.
1701  //
1702  if(!defines) {
1703  local_defines_.reset(new preproc_map);
1704  defines = local_defines_.get();
1705  }
1706 
1707  buf_.reset(new preprocessor_streambuf(defines));
1708 
1709  // Begin processing.
1710  buf_->add_preprocessor<preprocessor_file>(fname);
1711 
1712  //
1713  // TODO: not sure if this call is needed. Previously, this call was passed a
1714  // preprocessor_streambuf pointer and the std::basic_istream constructor was
1715  // called with its contents. However, at that point the preprocessing should
1716  // already have completed, meaning this call might be redundant. Not sure.
1717  //
1718  // - vultraz, 2017-08-31
1719  //
1720  init(buf_.get());
1721  }
1722 
1724  {
1725  clear(std::ios_base::goodbit);
1726  exceptions(std::ios_base::goodbit);
1727  rdbuf(nullptr);
1728  }
1729 
1730  std::unique_ptr<preprocessor_streambuf> buf_;
1731  std::unique_ptr<preproc_map> local_defines_;
1732 };
1733 
1734 
1735 // ==================================================================================
1736 // FREE-STANDING FUNCTIONS
1737 // ==================================================================================
1738 
1739 filesystem::scoped_istream preprocess_file(const std::string& fname, preproc_map* defines)
1740 {
1741  log_scope("preprocessing file " + fname + " ...");
1742 
1743  // NOTE: the preprocessor_scope_helper does *not* take ownership of defines.
1744  return filesystem::scoped_istream(new preprocessor_scope_helper(fname, defines));
1745 }
1746 
1747 std::string preprocess_string(const std::string& contents, preproc_map* defines, const std::string& textdomain)
1748 {
1749  log_scope("preprocessing string " + contents.substr(0, 10) + " ...");
1750 
1751  std::unique_ptr<preprocessor_streambuf> buf;
1752  std::unique_ptr<preproc_map> local_defines;
1753 
1754  //
1755  // If no defines were provided, we create a new local preproc_map and assign
1756  // it to defines temporarily. In this case, the map will be deleted once this
1757  // object is destroyed and defines will still be subsequently null.
1758  //
1759  if(!defines) {
1760  local_defines.reset(new preproc_map);
1761  defines = local_defines.get();
1762  }
1763 
1764  buf.reset(new preprocessor_streambuf(defines));
1765 
1766  // Begin processing.
1767  buf->add_preprocessor<preprocessor_data>(
1768  std::unique_ptr<std::istream>(new std::istringstream(contents)), "<string>", "", 1, game_config::path, textdomain, nullptr);
1769  return formatter() << buf.get();
1770 }
1771 
1772 void preprocess_resource(const std::string& res_name,
1773  preproc_map* defines_map,
1774  bool write_cfg,
1775  bool write_plain_cfg,
1776  const std::string& parent_directory)
1777 {
1778  if(filesystem::is_directory(res_name)) {
1779  std::vector<std::string> dirs, files;
1780 
1783 
1784  // Subdirectories
1785  for(const std::string& dir : dirs) {
1786  LOG_PREPROC << "processing sub-dir: " << dir;
1787  preprocess_resource(dir, defines_map, write_cfg, write_plain_cfg, parent_directory);
1788  }
1789 
1790  // Files in current directory
1791  for(const std::string& file : files) {
1792  if(filesystem::is_cfg(file)) {
1793  preprocess_resource(file, defines_map, write_cfg, write_plain_cfg, parent_directory);
1794  }
1795  }
1796 
1797  return;
1798  }
1799 
1800  LOG_PREPROC << "processing resource: " << res_name;
1801 
1802  // disable filename encoding to get clear #line in cfg.plain
1803  encode_filename = false;
1804 
1805  filesystem::scoped_istream stream = preprocess_file(res_name, defines_map);
1806 
1807  std::stringstream ss;
1808 
1809  // Set the failbit so if we get any preprocessor exceptions (e.g.:preproc_config::error)
1810  // they will be propagated in the main program, instead of just setting the
1811  // failbit on the stream. This was necessary in order for the MSVC and GCC
1812  // binaries to behave the same way.
1813  ss.exceptions(std::ios_base::failbit);
1814 
1815  ss << (*stream).rdbuf();
1816 
1817  LOG_PREPROC << "processing finished";
1818 
1819  if(write_cfg || write_plain_cfg) {
1820  std::string streamContent = ss.str();
1821  config cfg = io::read(streamContent);
1822 
1823  const std::string preproc_res_name = parent_directory + "/" + filesystem::base_name(res_name);
1824 
1825  // Write the processed cfg file
1826  if(write_cfg) {
1827  LOG_PREPROC << "writing cfg file: " << preproc_res_name;
1828 
1830  io::write(*filesystem::ostream_file(preproc_res_name), cfg);
1831  }
1832 
1833  // Write the plain cfg file
1834  if(write_plain_cfg) {
1835  LOG_PREPROC << "writing plain cfg file: " << (preproc_res_name + ".plain");
1836 
1838  filesystem::write_file(preproc_res_name + ".plain", streamContent);
1839  }
1840  }
1841 }
map_location loc
Definition: move.cpp:172
double t
Definition: astarsearch.cpp:63
unsigned in
If equal to search_counter, the node is off the list.
Definition: astarsearch.cpp:70
Helper class for buffering a std::istream.
Helper class for buffering a std::istream.
bool eof() const
Is the end of input reached?
int get()
Gets and consumes a character from the buffer.
int peek()
Gets a character from the buffer.
Class for writing a config out to a file in pieces.
void close_child(const std::string &key)
void write_key_val(const std::string &key, const T &value)
This template function will work with any type that can be assigned to an attribute_value.
void open_child(const std::string &key)
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:158
bool has_attribute(config_key_type key) const
Definition: config.cpp:157
child_itors child_range(config_key_type key)
Definition: config.cpp:268
optional_config_impl< config > optional_child(config_key_type key, int n=0)
Equivalent to mandatory_child, but returns an empty optional if the nth child was not found.
Definition: config.cpp:380
std::ostringstream wrapper.
Definition: formatter.hpp:40
Specialized preprocessor for handling any kind of input stream.
std::unique_ptr< std::map< std::string, std::string > > local_defines_
Mapping of macro arguments to their content.
int skipping_
Non-zero when the preprocessor has to skip some input text.
virtual preprocessor::MODE parse_mode() override
Returns the appropriate parsing mode for this preprocessor.
void conditional_skip(bool skip)
int slowpath_
Set to true whenever input tokens cannot be directly sent to the target buffer.
std::string read_rest_of_line()
friend bool operator!=(preprocessor_data::token_desc::token_type, char)
std::vector< std::string > strings_
Buffer for delayed input processing.
filesystem::scoped_istream in_scope_
Manages the lifetime of the std::istream pointer we own.
preprocessor_data(preprocessor_streambuf &, filesystem::scoped_istream, const std::string &history, const std::string &name, int line, const std::string &dir, const std::string &domain, std::unique_ptr< std::map< std::string, std::string >> defines, bool is_define=false)
buffered_istream in_
Input stream.
friend bool operator==(preprocessor_data::token_desc::token_type, char)
bool is_define_
True iff we are currently parsing a macros content, otherwise false.
std::vector< token_desc > tokens_
Stack of nested preprocessing chunks.
std::string read_line()
std::string read_word()
virtual bool get_chunk() override
Preprocesses and sends some text to the parent_ buffer.
std::string directory_
void push_token(token_desc::token_type)
Specialized preprocessor for handling a file or a set of files.
std::vector< std::string >::const_iterator pos_
std::vector< std::string > files_
virtual bool get_chunk() override
Inserts and processes the next file in the list of included files.
const std::string & name_
std::vector< std::string >::const_iterator end_
virtual void init() override
Allows specifying any actions that need to be called after the constructor completes.
preprocessor_file(preprocessor_streambuf &t, const std::string &name, std::size_t symbol_index=-1)
Constructor.
Target for sending preprocessed output.
std::stringstream buffer_
Buffer filled by the current preprocessor.
preprocessor * current() const
bool quoted_
Set to true if one preprocessor for this target started to read a string.
preprocessor_streambuf(preproc_map *def)
preproc_map default_defines_
virtual int underflow() override
Inherited from basic_streambuf.
void restore_old_preprocessor()
Restores the old preprocessing context.
std::string out_buffer_
Buffer read by the STL stream.
std::string get_current_file()
Decodes the filenames placed in a location.
std::deque< std::unique_ptr< preprocessor > > preprocessor_queue_
Input preprocessor queue.
void add_preprocessor(A &&... args)
void error(const std::string &, int)
preprocessor_streambuf(const preprocessor_streambuf &t)
void warning(const std::string &, int)
Base class for preprocessing an input.
virtual MODE parse_mode()
Returns the appropriate parsing mode for this preprocessor.
std::string old_textdomain_
virtual void init()
Allows specifying any actions that need to be called after the constructor completes.
virtual ~preprocessor()
std::string old_location_
friend class preprocessor_streambuf
preprocessor(preprocessor_streambuf &t)
Sets up a new preprocessor for stream buffer t.
virtual bool get_chunk()=0
Preprocesses and sends some text to the parent_ buffer.
preprocessor_streambuf & parent_
Represents version numbers.
std::string str() const
Serializes the version number into string form.
Definitions for the interface to Wesnoth Markup Language (WML).
std::string deprecated_message(const std::string &elem_name, DEP_LEVEL level, const version_info &version, const std::string &detail)
Definition: deprecation.cpp:29
DEP_LEVEL
See https://wiki.wesnoth.org/CompatibilityStandards for more info.
Definition: deprecation.hpp:21
const config * cfg
std::size_t i
Definition: function.cpp:1032
bool do_version_check(const version_info &a, VERSION_COMP_OP op, const version_info &b)
VERSION_COMP_OP parse_version_op(const std::string &op_str)
Interfaces for manipulating version numbers of engine, add-ons, etc.
VERSION_COMP_OP
@ OP_INVALID
Standard logging facilities (interface).
#define log_scope(description)
Definition: log.hpp:275
void clear()
Clear the current render target.
Definition: draw.cpp:42
void line(int from_x, int from_y, int to_x, int to_y)
Draw a line.
Definition: draw.cpp:189
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, name_mode mode, filter_mode filter, reorder_mode reorder, file_tree_checksum *checksum)
Get a list of all files and/or directories in a given directory.
Definition: filesystem.cpp:463
bool is_cfg(const std::string &filename)
Returns true if the file ends with the wmlfile extension.
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
bool is_directory(const std::string &fname)
Returns true if the given file is a directory.
utils::optional< std::string > get_wml_location(const std::string &path, const utils::optional< std::string > &current_dir)
Returns a translated path to the actual file or directory, if it exists.
filesystem::scoped_ostream ostream_file(const std::string &fname, std::ios_base::openmode mode, bool create_directory)
static bool create_directory_if_missing_recursive(const bfs::path &dirpath)
Definition: filesystem.cpp:399
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:53
void write_file(const std::string &fname, const std::string &data, std::ios_base::openmode mode)
Throws io_exception if an error occurs.
std::string get_short_wml_path(const std::string &filename)
Returns a short path to filename, skipping the (user) data directory.
std::string directory_name(const std::string &file)
Returns the directory name of a file, with filename stripped.
std::string path
Definition: filesystem.cpp:106
const version_info wesnoth_version(VERSION)
config read(std::istream &in, abstract_validator *validator)
Definition: parser.cpp:600
void write(std::ostream &out, const configr_of &cfg, unsigned int level)
Definition: parser.cpp:737
logger & err()
Definition: log.cpp:339
struct utils::detail::formula_initer init
std::vector< std::string > quoted_split(const std::string &val, char c, int flags, char quote)
This function is identical to split(), except it does not split when it otherwise would if the previo...
int stoi(std::string_view str)
Same interface as std::stoi and meant as a drop in replacement, except:
Definition: charconv.hpp:155
bool portable_isspace(const char c)
std::string escape(std::string_view str, const char *special_chars)
Prepends a configurable set of characters with a backslash.
std::vector< std::string > split(const config_attribute_value &val)
std::string::const_iterator iterator
Definition: tokenizer.hpp:25
#define DBG_PREPROC
static lg::log_domain log_preprocessor("preprocessor")
std::string preprocess_string(const std::string &contents, preproc_map *defines, const std::string &textdomain)
Function to use the WML preprocessor on a string.
static const std::string left_curly_str
static const std::string current_dir_str
#define ERR_PREPROC
std::string lineno_string(const std::string &lineno)
void preprocess_resource(const std::string &res_name, preproc_map *defines_map, bool write_cfg, bool write_plain_cfg, const std::string &parent_directory)
#define WRN_PREPROC
static std::string get_file_code(const std::string &filename)
static bool encode_filename
static const std::string current_file_str
bool operator==(preprocessor_data::token_desc::token_type, char)
static std::string get_filename(const std::string &file_code)
static std::string get_location(const std::string &loc)
std::ostream & operator<<(std::ostream &stream, const preproc_define &def)
#define LOG_PREPROC
static std::map< std::string, int > file_number_map
static const std::string right_curly_str
bool operator!=(preprocessor_data::token_desc::token_type rhs, char lhs)
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
Function to use the WML preprocessor on a file.
static std::string preprocessor_error_detail_prefix
std::map< std::string, struct preproc_define > preproc_map
constexpr unsigned char INLINED_PREPROCESS_DIRECTIVE_CHAR
Definition: tokenizer.hpp:29
std::string filename
Filename.
void write(config_writer &, const std::string &) const
std::string value
version_info deprecation_version
bool operator<(const preproc_define &) const
std::string textdomain
void write_argument(config_writer &, const std::string &) const
void read(const config &)
void read_argument(const config &)
std::string deprecation_message
std::vector< std::string > arguments
static void insert(preproc_map &, const config &)
std::map< std::string, std::string > optional_arguments
bool operator==(const preproc_define &) const
utils::optional< DEP_LEVEL > deprecation_level
std::string location
bool is_deprecated() const
Description of a preprocessing chunk.
token_desc(token_type type, const int stack_pos, const int linenum)
int stack_pos
Starting position in strings_ of the delayed text for this chunk.
std::unique_ptr< preprocessor_streambuf > buf_
std::unique_ptr< preproc_map > local_defines_
preprocessor_scope_helper(const std::string &fname, preproc_map *defines)
contains the current text being parsed as well as the token_type of what's being parsed.
Definition: tokenizer.hpp:41
token_type type
Definition: tokenizer.hpp:83
mock_char c
mock_party p
static map_location::direction n
static map_location::direction s
Some defines: VERSION, PACKAGE, MIN_SAVEGAME_VERSION.
#define PACKAGE
Definition: wesconfig.h:23
#define d
#define e
#define b