The Battle for Wesnoth  1.13.11+dev
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
preprocessor.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2003 by David White <dave@whitevine.net>
3  Copyright (C) 2005 - 2018 by Guillaume Melquiond <guillaume.melquiond@gmail.com>
4  Part of the Battle for Wesnoth Project http://www.wesnoth.org/
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation; either version 2 of the License, or
9  (at your option) any later version.
10  This program is distributed in the hope that it will be useful,
11  but WITHOUT ANY WARRANTY.
12 
13  See the COPYING file for more details.
14 */
15 
16 /**
17  * @file
18  * WML preprocessor.
19  */
20 
21 #include "buffered_istream.hpp"
22 #include "config.hpp"
23 #include "filesystem.hpp"
24 #include "game_config.hpp"
25 #include "log.hpp"
27 #include "serialization/parser.hpp"
29 #include "version.hpp"
30 #include "wesconfig.h"
31 #include "deprecation.hpp"
32 
33 #include <stdexcept>
34 #include <deque>
35 
36 static lg::log_domain log_preprocessor("preprocessor");
37 #define ERR_PREPROC LOG_STREAM(err, log_preprocessor)
38 #define WRN_PREPROC LOG_STREAM(warn, log_preprocessor)
39 #define LOG_PREPROC LOG_STREAM(info, log_preprocessor)
40 #define DBG_PREPROC LOG_STREAM(debug, log_preprocessor)
41 
42 static std::string current_file_str = "CURRENT_FILE";
43 static std::string current_dir_str = "CURRENT_DIRECTORY";
44 
45 // map associating each filename encountered to a number
46 static std::map<std::string, int> file_number_map;
47 
48 static bool encode_filename = true;
49 
51 
52 static const char OUTPUT_SEPARATOR = '\xFE';
53 
54 // get filename associated to this code
55 static std::string get_filename(const std::string& file_code)
56 {
57  if(!encode_filename) {
58  return file_code;
59  }
60 
61  std::stringstream s;
62  s << file_code;
63  int n = 0;
64  s >> std::hex >> n;
65 
66  for(const auto& p : file_number_map) {
67  if(p.second == n) {
68  return p.first;
69  }
70  }
71 
72  return "<unknown>";
73 }
74 
75 // Get code associated to this filename
76 static std::string get_file_code(const std::string& filename)
77 {
78  if(!encode_filename) {
79  return filename;
80  }
81 
82  // Current number of encountered filenames
83  static int current_file_number = 0;
84 
85  int& fnum = file_number_map[utils::escape(filename, " \\")];
86  if(fnum == 0) {
87  fnum = ++current_file_number;
88  }
89 
90  std::ostringstream shex;
91  shex << std::hex << fnum;
92 
93  return shex.str();
94 }
95 
96 // decode the filenames placed in a location
98 {
99  std::string res;
100  std::vector<std::string> pos = utils::quoted_split(loc, ' ');
101 
102  if(pos.empty()) {
103  return res;
104  }
105 
106  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
107  while(true) {
108  res += get_filename(*(i++));
109 
110  if(i == end) {
111  break;
112  }
113 
114  res += ' ';
115  res += *(i++);
116 
117  if(i == end) {
118  break;
119  }
120 
121  res += ' ';
122  }
123 
124  return res;
125 }
126 
127 
128 // ==================================================================================
129 // PREPROC_DEFINE IMPLEMENTATION
130 // ==================================================================================
131 
133 {
134  return value == v.value && arguments == v.arguments;
135 }
136 
138 {
139  if(location < v.location) {
140  return true;
141  }
142 
143  if(v.location < location) {
144  return false;
145  }
146 
147  return linenum < v.linenum;
148 }
149 
151 {
152  const std::string key = "argument";
153 
154  writer.open_child(key);
155 
156  writer.write_key_val("name", arg);
157  writer.close_child(key);
158 }
159 
161 {
162  const std::string key = "preproc_define";
163  writer.open_child(key);
164 
165  writer.write_key_val("name", name);
166  writer.write_key_val("value", value);
167  writer.write_key_val("textdomain", textdomain);
168  writer.write_key_val("linenum", std::to_string(linenum));
169  writer.write_key_val("location", get_location(location));
170 
171  for(const std::string& arg : arguments) {
172  write_argument(writer, arg);
173  }
174 
175  writer.close_child(key);
176 }
177 
179 {
180  arguments.push_back(cfg["name"]);
181 }
182 
183 void preproc_define::read(const config& cfg)
184 {
185  value = cfg["value"].str();
186  textdomain = cfg["textdomain"].str();
187  linenum = cfg["linenum"];
188  location = cfg["location"].str();
189 
190  for(const config& arg : cfg.child_range("argument")) {
191  read_argument(arg);
192  }
193 }
194 
195 preproc_map::value_type preproc_define::read_pair(const config& cfg)
196 {
197  preproc_define second;
198  second.read(cfg);
199 
200  return preproc_map::value_type(cfg["name"], second);
201 }
202 
203 std::ostream& operator<<(std::ostream& stream, const preproc_define& def)
204 {
205  return stream << "value: " << def.value << " arguments: " << def.location;
206 }
207 
208 std::ostream& operator<<(std::ostream& stream, const preproc_map::value_type& def)
209 {
210  return stream << def.second;
211 }
212 
213 // ==================================================================================
214 // PREPROCESSOR BASE
215 // ==================================================================================
216 
218 
219 /**
220  * Base class for preprocessing an input.
221  */
223 {
225 
226 protected:
227  /**
228  * Sets up a new preprocessor for stream buffer \a t.
229  * Saves the current preprocessing context of #parent_. It will be automatically restored on destruction.
230  *
231  * It relies on preprocessor_streambuf so it's implemented after that class is declared.
232  */
234 
236 
237 public:
238  virtual ~preprocessor()
239  {
240  }
241 
242  /** Allows specifying any actions that need to be called after the constructor completes. */
243  virtual void init()
244  {
245  }
246 
247  /**
248  * Preprocesses and sends some text to the #parent_ buffer.
249  * @return false when the input has no data left.
250  */
251  virtual bool get_chunk() = 0;
252 
254 
255  /** Returns the appropriate parsing mode for this preprocessor. */
256  virtual MODE parse_mode()
257  {
258  return NO_PARSING;
259  }
260 
261 private:
264 
266 };
267 
268 
269 // ==================================================================================
270 // PREPROCESSOR BUFFER
271 // ==================================================================================
272 
273 /**
274  * Target for sending preprocessed output.
275  * Objects of this class can be plugged into an STL stream.
276  */
277 class preprocessor_streambuf : public std::streambuf
278 {
279 public:
281  : std::streambuf()
282  , out_buffer_("")
283  , buffer_()
285  , defines_(def)
286  , default_defines_()
288  , location_("")
289  , linenum_(0)
290  , quoted_(false)
291  {
292  }
293 
294  /** Decodes the filenames placed in a location. */
296 
297  void error(const std::string&, int);
298  void warning(const std::string&, int);
299 
300  template<typename T, typename... A>
301  void add_preprocessor(A&&... args)
302  {
303  preprocessor_queue_.emplace_back(new T(*this, std::forward<A>(args)...));
304  preprocessor_queue_.back()->init();
305  }
306 
308  {
309  preprocessor_queue_.pop_back();
310  }
311 
312  int depth() const
313  {
314  return preprocessor_queue_.size();
315  }
316 
318  {
319  return preprocessor_queue_.empty() ? nullptr : preprocessor_queue_.back().get();
320  }
321 
322 private:
324  : std::streambuf()
325  , out_buffer_("")
326  , buffer_()
328  , defines_(t.defines_)
329  , default_defines_()
331  , location_("")
332  , linenum_(0)
333  , quoted_(t.quoted_)
334  {
335  }
336 
337  /** Inherited from basic_streambuf. */
338  virtual int underflow() override;
339 
341 
342  /** Buffer read by the STL stream. */
344 
345  /** Buffer filled by the #current_ preprocessor. */
346  std::stringstream buffer_;
347 
348  /** Input preprocessor queue. */
349  std::deque<std::unique_ptr<preprocessor>> preprocessor_queue_;
350 
353 
356 
357  int linenum_;
358 
359  /**
360  * Set to true if one preprocessor for this target started to read a string.
361  * Deeper-nested preprocessors are then forbidden to.
362  */
363  bool quoted_;
364 
365  friend class preprocessor;
366  friend class preprocessor_file;
367  friend class preprocessor_data;
369 };
370 
371 /** Preprocessor constructor. */
373  : parent_(t)
374  , old_textdomain_(t.textdomain_)
375  , old_location_(t.location_)
376  , old_linenum_(t.linenum_)
377 {
378 }
379 
380 /**
381  * Called by an STL stream whenever it has reached the end of #out_buffer_.
382  * Fills #buffer_ by calling the #current_ preprocessor, then copies its
383  * content into #out_buffer_.
384  * @return the first character of #out_buffer_ if any, EOF otherwise.
385  */
387 {
388  unsigned sz = 0;
389  if(char* gp = gptr()) {
390  if(gp < egptr()) {
391  // Sanity check: the internal buffer has not been totally consumed,
392  // should we force the caller to use what remains first?
393  return *gp;
394  }
395 
396  // The buffer has been completely read; fill it again.
397  // Keep part of the previous buffer, to ensure putback capabilities.
398  sz = out_buffer_.size();
399  buffer_.str(std::string());
400 
401  if(sz > 3) {
402  buffer_ << out_buffer_.substr(sz - 3);
403  sz = 3;
404  } else {
405  buffer_ << out_buffer_;
406  }
407  } else {
408  // The internal get-data pointer is null
409  }
410 
411  const int desired_fill_amount = 2000;
412 
413  while(current() && buffer_.rdbuf()->in_avail() < desired_fill_amount) {
414  // Process files and data chunks until the desired buffer size is reached
415  if(!current()->get_chunk()) {
416  // Drop the current preprocessor item from the queue.
418  }
419  }
420 
421  // Update the internal state and data pointers
422  out_buffer_ = buffer_.str();
423  if(out_buffer_.empty()) {
424  return EOF;
425  }
426 
427  char* begin = &*out_buffer_.begin();
428  unsigned bs = out_buffer_.size();
429 
430  setg(begin, begin + sz, begin + bs);
431 
432  if(sz >= bs) {
433  return EOF;
434  }
435 
436  return static_cast<unsigned char>(*(begin + sz));
437 }
438 
439 /**
440 * Restores the old preprocessing context.
441 * Appends location and domain directives to the buffer, so that the parser
442 * notices these changes.
443 */
445 {
446  preprocessor* current = this->current();
447 
448  if(!current->old_location_.empty()) {
449  buffer_ << OUTPUT_SEPARATOR << "line " << current->old_linenum_ << ' ' << current->old_location_ << '\n';
450  }
451 
452  if(!current->old_textdomain_.empty() && textdomain_ != current->old_textdomain_) {
453  buffer_ << OUTPUT_SEPARATOR << "textdomain " << current->old_textdomain_ << '\n';
454  }
455 
456  location_ = current->old_location_;
457  linenum_ = current->old_linenum_;
458  textdomain_ = current->old_textdomain_;
459 
460  // Drop the preprocessor from the queue.
462 }
463 
465 {
466  unsigned nested_level = 0;
467 
468  preprocessor* pre = nullptr;
469 
470  // Iterate backwards over queue to get the last non-macro preprocessor.
471  for(auto p = preprocessor_queue_.rbegin(); p != preprocessor_queue_.rend(); ++p) {
472  pre = p->get();
473 
474  if(!pre || pre->parse_mode() == preprocessor::PARSES_FILE) {
475  break;
476  }
477 
478  if(pre->parse_mode() == preprocessor::PARSES_MACRO) {
479  ++nested_level;
480  }
481  }
482 
483  std::string res;
484  std::vector<std::string> pos = utils::quoted_split(location_, ' ');
485 
486  if(pos.size() <= 2 * nested_level) {
487  return res;
488  }
489 
490  return get_filename(pos[2 * nested_level]);
491 }
492 
494 {
495  std::vector<std::string> pos = utils::quoted_split(lineno, ' ');
496  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
497  std::string included_from = preprocessor_error_detail_prefix + "included from ";
498  std::string res;
499 
500  while(i != end) {
501  const std::string& line = *(i++);
502 
503  if(!res.empty()) {
504  res += included_from;
505  }
506 
507  if(i != end) {
508  res += get_filename(*(i++));
509  } else {
510  res += "<unknown>";
511  }
512 
513  res += ':' + line;
514  }
515 
516  if(res.empty()) {
517  res = "???";
518  }
519 
520  return res;
521 }
522 
523 void preprocessor_streambuf::error(const std::string& error_type, int l)
524 {
525  std::string position, error;
526  std::ostringstream pos;
527 
528  pos << l << ' ' << location_;
529  position = lineno_string(pos.str());
530 
531  error = error_type + '\n';
532  error += "at " + position;
533 
534  ERR_PREPROC << error << '\n';
535 
536  throw preproc_config::error(error);
537 }
538 
539 void preprocessor_streambuf::warning(const std::string& warning_type, int l)
540 {
541  std::string position, warning;
542  std::ostringstream pos;
543 
544  pos << l << ' ' << location_;
545  position = lineno_string(pos.str());
546 
547  warning = warning_type + '\n';
548  warning += "at " + position;
549 
550  WRN_PREPROC << warning << '\n';
551 }
552 
553 
554 // ==================================================================================
555 // PREPROCESSOR FILE
556 // ==================================================================================
557 
558 /**
559  * Specialized preprocessor for handling a file or a set of files.
560  * A preprocessor_file object is created when a preprocessor encounters an
561  * inclusion directive that resolves to a file or directory, e.g. '{themes/}'.
562  */
564 {
565 public:
566  /** Constructor. It relies on preprocessor_data so it's implemented after that class is declared. */
567  preprocessor_file(preprocessor_streambuf& t, const std::string& name, size_t symbol_index = -1);
568 
569  virtual void init() override;
570 
571  /**
572  * Inserts and processes the next file in the list of included files.
573  * @return false if there is no next file.
574  */
575  virtual bool get_chunk() override
576  {
577  while(pos_ != end_) {
578  const std::string& name = *(pos_++);
579  unsigned sz = name.size();
580 
581  // Use reverse iterator to optimize testing
582  if(sz < 5 || !std::equal(name.rbegin(), name.rbegin() + 4, "gfc.")) {
583  continue;
584  }
585 
587  return true;
588  }
589 
590  return false;
591  }
592 
593 private:
594  std::vector<std::string> files_;
595  std::vector<std::string>::const_iterator pos_, end_;
596 
598 
600 };
601 
602 
603 // ==================================================================================
604 // PREPROCESSOR DATA
605 // ==================================================================================
606 
607 /**
608  * Specialized preprocessor for handling any kind of input stream.
609  * This is the core of the preprocessor.
610  */
612 {
613  /** Description of a preprocessing chunk. */
614  struct token_desc
615  {
616  enum TOKEN_TYPE {
617  START, // Toplevel
618  PROCESS_IF, // Processing the "if" branch of a ifdef/ifndef (the "else" branch will be skipped)
619  PROCESS_ELSE, // Processing the "else" branch of a ifdef/ifndef
620  SKIP_IF, // Skipping the "if" branch of a ifdef/ifndef (the "else" branch, if any, will be processed)
621  SKIP_ELSE, // Skipping the "else" branch of a ifdef/ifndef
622  STRING, // Processing a string
623  VERBATIM, // Processing a verbatim string
624  MACRO_SPACE, // Processing between chunks of a macro call (skip spaces)
625  MACRO_CHUNK, // Processing inside a chunk of a macro call (stop on space or '(')
626  MACRO_PARENS // Processing a parenthesized macro argument
627  };
628 
629  token_desc(TOKEN_TYPE type, const int stack_pos, const int linenum)
630  : type(type)
631  , stack_pos(stack_pos)
632  , linenum(linenum)
633  {
634  }
635 
637 
638  /** Starting position in #strings_ of the delayed text for this chunk. */
640  int linenum;
641  };
642 
643  /**
644  * Manages the lifetime of the @c std::istream pointer we own.
645  *
646  * Since @ref in_ uses the stream as well this object must be created
647  * before @ref in_ and destroyed after @ref in_ is destroyed.
648  */
650 
651  /** Input stream. */
653 
655 
656  /** Buffer for delayed input processing. */
657  std::vector<std::string> strings_;
658 
659  /** Mapping of macro arguments to their content. */
660  std::unique_ptr<std::map<std::string, std::string>> local_defines_;
661 
662  /** Stack of nested preprocessing chunks. */
663  std::vector<token_desc> tokens_;
664 
665  /**
666  * Set to true whenever input tokens cannot be directly sent to the target
667  * buffer. For instance, this happens with macro arguments. In that case,
668  * the output is redirected toward #strings_ until it can be processed.
669  */
671 
672  /**
673  * Non-zero when the preprocessor has to skip some input text.
674  * Increased whenever entering a conditional branch that is not useful,
675  * e.g. a ifdef that evaluates to false.
676  */
678  int linenum_;
679 
680  /** True iff we are currently parsing a macros content, otherwise false. */
682 
686 
687  void skip_spaces();
688  void skip_eol();
690  void pop_token();
691  void put(char);
692  void put(const std::string& /*, int change_line = 0 */);
693  void conditional_skip(bool skip);
694 
695 public:
698  const std::string& history,
699  const std::string& name,
700  int line,
701  const std::string& dir,
702  const std::string& domain,
703  std::unique_ptr<std::map<std::string, std::string>> defines,
704  bool is_define = false);
705 
706  virtual bool get_chunk() override;
707 
708  virtual preprocessor::MODE parse_mode() override
709  {
710  return is_define_ ? PARSES_MACRO : PARSES_FILE;
711  }
712 
717 };
718 
720 {
721  throw std::logic_error("don't compare tokens with characters");
722 }
723 
725 {
726  return rhs == lhs;
727 }
728 
730 {
731  return !(lhs == rhs);
732 }
733 
735 {
736  return rhs != lhs;
737 }
738 
739 /** preprocessor_file constructor. */
741  : preprocessor(t)
742  , files_()
743  , pos_()
744  , end_()
745  , name_(name)
746  , is_directory_(filesystem::is_directory(name))
747 {
748  if(is_directory_) {
749  filesystem::get_files_in_dir(name, &files_, nullptr,
753  );
754 
755  for(const std::string& fname : files_) {
756  size_t cpos = fname.rfind(" ");
757 
758  if(cpos != std::string::npos && cpos >= symbol_index) {
759  std::stringstream ss;
760  ss << "Found filename containing whitespace: '" << filesystem::base_name(fname)
761  << "' in included directory '" << name << "'.\nThe included symbol probably looks similar to '"
762  << filesystem::directory_name(fname.substr(symbol_index)) << "'";
763 
764  // TODO: find a real linenumber
765  parent_.error(ss.str(), -1);
766  }
767  }
768  } else {
769  // Handled in the init() function.
770  }
771 
772  pos_ = files_.begin();
773  end_ = files_.end();
774 }
775 
777 {
778  if(is_directory_) {
779  return;
780  }
781 
783 
784  if(!file_stream->good()) {
785  ERR_PREPROC << "Could not open file " << name_ << std::endl;
786  } else {
787  parent_.add_preprocessor<preprocessor_data>(std::move(file_stream), "",
790  );
791  }
792 }
793 
796  const std::string& history,
797  const std::string& name,
798  int linenum,
799  const std::string& directory,
800  const std::string& domain,
801  std::unique_ptr<std::map<std::string, std::string>> defines,
802  bool is_define)
803  : preprocessor(t)
804  , in_scope_(std::move(i))
805  , in_(*in_scope_)
806  , directory_(directory)
807  , strings_()
808  , local_defines_(std::move(defines))
809  , tokens_()
810  , slowpath_(0)
811  , skipping_(0)
812  , linenum_(linenum)
813  , is_define_(is_define)
814 {
815  std::ostringstream s;
816  s << history;
817 
818  if(!name.empty()) {
819  if(!history.empty()) {
820  s << ' ';
821  }
822 
823  s << get_file_code(name);
824  }
825 
826  if(!t.location_.empty()) {
827  s << ' ' << t.linenum_ << ' ' << t.location_;
828  }
829 
830  t.location_ = s.str();
831  t.linenum_ = linenum;
832 
833  t.buffer_ << OUTPUT_SEPARATOR << "line " << linenum << ' ' << t.location_ << '\n';
834 
835  if(t.textdomain_ != domain) {
836  t.buffer_ << OUTPUT_SEPARATOR << "textdomain " << domain << '\n';
837  t.textdomain_ = domain;
838  }
839 
841 }
842 
844 {
845  tokens_.emplace_back(t, strings_.size(), linenum_);
846 
847  if(t == token_desc::MACRO_SPACE) {
848  // Macro expansions do not have any associated storage at start.
849  return;
850  } else if(t == token_desc::STRING || t == token_desc::VERBATIM) {
851  /* Quoted strings are always inlined in the parent token. So
852  * they need neither storage nor metadata, unless the parent
853  * token is a macro expansion.
854  */
855  token_desc::TOKEN_TYPE& outer_type = tokens_[tokens_.size() - 2].type;
856  if(outer_type != token_desc::MACRO_SPACE) {
857  return;
858  }
859 
860  outer_type = token_desc::MACRO_CHUNK;
861  tokens_.back().stack_pos = strings_.size() + 1;
862  }
863 
864  std::ostringstream s;
865  if(!skipping_ && slowpath_) {
866  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
867  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
868  }
869 
870  strings_.push_back(s.str());
871 }
872 
874 {
875  token_desc::TOKEN_TYPE inner_type = tokens_.back().type;
876  unsigned stack_pos = tokens_.back().stack_pos;
877 
878  tokens_.pop_back();
879 
880  token_desc::TOKEN_TYPE& outer_type = tokens_.back().type;
881 
882  if(inner_type == token_desc::MACRO_PARENS) {
883  // Parenthesized macro arguments are left on the stack.
884  assert(outer_type == token_desc::MACRO_SPACE);
885  return;
886  }
887 
888  if(inner_type == token_desc::STRING || inner_type == token_desc::VERBATIM) {
889  // Quoted strings are always inlined.
890  assert(stack_pos == strings_.size());
891  return;
892  }
893 
894  if(outer_type == token_desc::MACRO_SPACE) {
895  /* A macro expansion does not have any associated storage.
896  * Instead, storage of the inner token is not discarded
897  * but kept as a new macro argument. But if the inner token
898  * was a macro expansion, it is about to be appended, so
899  * prepare for it.
900  */
901  if(inner_type == token_desc::MACRO_SPACE || inner_type == token_desc::MACRO_CHUNK) {
902  strings_.erase(strings_.begin() + stack_pos, strings_.end());
903  strings_.emplace_back();
904  }
905 
906  assert(stack_pos + 1 == strings_.size());
907  outer_type = token_desc::MACRO_CHUNK;
908 
909  return;
910  }
911 
912  strings_.erase(strings_.begin() + stack_pos, strings_.end());
913 }
914 
916 {
917  for(;;) {
918  int c = in_.peek();
919 
920  if(in_.eof() || (c != ' ' && c != '\t')) {
921  return;
922  }
923 
924  in_.get();
925  }
926 }
927 
929 {
930  for(;;) {
931  int c = in_.get();
932 
933  if(c == '\n') {
934  ++linenum_;
935  return;
936  }
937 
938  if(in_.eof()) {
939  return;
940  }
941  }
942 }
943 
945 {
946  std::string res;
947 
948  for(;;) {
949  int c = in_.peek();
950 
951  if(c == preprocessor_streambuf::traits_type::eof() || utils::portable_isspace(c)) {
952  // DBG_PREPROC << "(" << res << ")\n";
953  return res;
954  }
955 
956  in_.get();
957  res += static_cast<char>(c);
958  }
959 }
960 
962 {
963  std::string res;
964 
965  for(;;) {
966  int c = in_.get();
967 
968  if(c == '\n') {
969  ++linenum_;
970  return res;
971  }
972 
973  if(in_.eof()) {
974  return res;
975  }
976 
977  if(c != '\r') {
978  res += static_cast<char>(c);
979  }
980  }
981 }
982 
984 {
985  std::string res;
986 
987  while(in_.peek() != '\n' && !in_.eof()) {
988  int c = in_.get();
989 
990  if(c != '\r') {
991  res += static_cast<char>(c);
992  }
993  }
994 
995  return res;
996 }
997 
999 {
1000  if(skipping_) {
1001  return;
1002  }
1003 
1004  if(slowpath_) {
1005  strings_.back() += c;
1006  return;
1007  }
1008 
1009  int cond_linenum = c == '\n' ? linenum_ - 1 : linenum_;
1010 
1011  if(unsigned diff = cond_linenum - parent_.linenum_) {
1012  parent_.linenum_ = cond_linenum;
1013 
1014  if(diff <= parent_.location_.size() + 11) {
1015  parent_.buffer_ << std::string(diff, '\n');
1016  } else {
1017  parent_.buffer_ << OUTPUT_SEPARATOR << "line " << parent_.linenum_ << ' ' << parent_.location_ << '\n';
1018  }
1019  }
1020 
1021  if(c == '\n') {
1022  ++parent_.linenum_;
1023  }
1024 
1025  parent_.buffer_ << c;
1026 }
1027 
1028 void preprocessor_data::put(const std::string& s /*, int line_change*/)
1029 {
1030  if(skipping_) {
1031  return;
1032  }
1033 
1034  if(slowpath_) {
1035  strings_.back() += s;
1036  return;
1037  }
1038 
1039  parent_.buffer_ << s;
1040  // parent_.linenum_ += line_change;
1041 }
1042 
1044 {
1045  if(skip) {
1046  ++skipping_;
1047  }
1048 
1050 }
1051 
1053 {
1054  char c = static_cast<char>(in_.get());
1055  token_desc& token = tokens_.back();
1056 
1057  if(in_.eof()) {
1058  // The end of file was reached.
1059  // Make sure we don't have any incomplete tokens.
1060  char const* s;
1061 
1062  switch(token.type) {
1063  case token_desc::START:
1064  return false; // everything is fine
1066  case token_desc::SKIP_IF:
1068  case token_desc::SKIP_ELSE:
1069  s = "#ifdef or #ifndef";
1070  break;
1071  case token_desc::STRING:
1072  s = "Quoted string";
1073  break;
1074  case token_desc::VERBATIM:
1075  s = "Verbatim string";
1076  break;
1079  s = "Macro substitution";
1080  break;
1082  s = "Macro argument";
1083  break;
1084  default:
1085  s = "???";
1086  }
1087 
1088  parent_.error(std::string(s) + " not terminated", token.linenum);
1089  }
1090 
1091  if(c == '\n') {
1092  ++linenum_;
1093  }
1094 
1095  if(c == OUTPUT_SEPARATOR) {
1096  std::string buffer(1, c);
1097 
1098  for(;;) {
1099  char d = static_cast<char>(in_.get());
1100 
1101  if(in_.eof() || d == '\n') {
1102  break;
1103  }
1104 
1105  buffer += d;
1106  }
1107 
1108  buffer += '\n';
1109  // line_change = 1-1 = 0
1110  put(buffer);
1111  } else if(token.type == token_desc::VERBATIM) {
1112  put(c);
1113 
1114  if(c == '>' && in_.peek() == '>') {
1115  put(in_.get());
1116  pop_token();
1117  }
1118  } else if(c == '<' && in_.peek() == '<') {
1119  in_.get();
1121  put('<');
1122  put('<');
1123  } else if(c == '"') {
1124  if(token.type == token_desc::STRING) {
1125  parent_.quoted_ = false;
1126  put(c);
1127  pop_token();
1128  } else if(!parent_.quoted_) {
1129  parent_.quoted_ = true;
1131  put(c);
1132  } else {
1133  parent_.error("Nested quoted string", linenum_);
1134  }
1135  } else if(c == '{') {
1137  ++slowpath_;
1138  } else if(c == ')' && token.type == token_desc::MACRO_PARENS) {
1139  pop_token();
1140  } else if(c == '#' && !parent_.quoted_) {
1141  std::string command = read_word();
1142  bool comment = false;
1143 
1144  if(command == "define") {
1145  skip_spaces();
1146  int linenum = linenum_;
1147  std::vector<std::string> items = utils::split(read_line(), ' ');
1148  std::map<std::string, std::string> optargs;
1149 
1150  if(items.empty()) {
1151  parent_.error("No macro name found after #define directive", linenum);
1152  }
1153 
1154  std::string symbol = items.front();
1155  items.erase(items.begin());
1156  int found_arg = 0, found_enddef = 0, found_deprecate = 0;
1157  int deprecation_level = -1;
1158  std::string buffer, deprecation_detail;
1159  version_info deprecation_version = game_config::wesnoth_version;
1160  for(;;) {
1161  if(in_.eof())
1162  break;
1163  char d = static_cast<char>(in_.get());
1164  if(d == '\n')
1165  ++linenum_;
1166  buffer += d;
1167  if(d == '#') {
1168  if(in_.peek() == 'a') {
1169  found_arg = 1;
1170  } else if(in_.peek() == 'd') {
1171  found_deprecate = 1;
1172  } else {
1173  found_enddef = 1;
1174  }
1175  } else {
1176  if(found_arg > 0 && ++found_arg == 4) {
1177  if(std::equal(buffer.end() - 3, buffer.end(), "arg")) {
1178  buffer.erase(buffer.end() - 4, buffer.end());
1179 
1180  skip_spaces();
1181  std::string argname = read_word();
1182  skip_eol();
1183 
1184  std::string argbuffer;
1185 
1186  int found_endarg = 0;
1187  for(;;) {
1188  if(in_.eof()) {
1189  break;
1190  }
1191 
1192  char e = static_cast<char>(in_.get());
1193  if(e == '\n') {
1194  ++linenum_;
1195  }
1196 
1197  argbuffer += e;
1198 
1199  if(e == '#') {
1200  found_endarg = 1;
1201  } else if(found_endarg > 0 && ++found_endarg == 7) {
1202  if(std::equal(argbuffer.end() - 6, argbuffer.end(), "endarg")) {
1203  argbuffer.erase(argbuffer.end() - 7, argbuffer.end());
1204  optargs[argname] = argbuffer;
1205  skip_eol();
1206  break;
1207  } else {
1208  parent_.error("Unterminated #arg definition", linenum_);
1209  }
1210  }
1211  }
1212  }
1213  }
1214 
1215  if(found_deprecate > 0 && ++found_deprecate == 11) {
1216  if(std::equal(buffer.end() - 10, buffer.end(), "deprecated")) {
1217  buffer.erase(buffer.end() - 11, buffer.end());
1218  skip_spaces();
1219  try {
1220  deprecation_level = std::max(deprecation_level, std::stoi(read_word()));
1221  } catch(std::invalid_argument&) {
1222  deprecation_level = 0;
1223  }
1224  deprecation_version = game_config::wesnoth_version;
1225  if(deprecation_level == 2 || deprecation_level == 3) {
1226  skip_spaces();
1227  deprecation_version = std::max(deprecation_version, version_info(read_word()));
1228  }
1229  skip_spaces();
1230  if(!deprecation_detail.empty()){
1231  deprecation_detail += '\n';
1232  }
1233  deprecation_detail = read_rest_of_line();
1234  }
1235  }
1236 
1237  if(found_enddef > 0 && ++found_enddef == 7) {
1238  if(std::equal(buffer.end() - 6, buffer.end(), "enddef")) {
1239  break;
1240  } else {
1241  found_enddef = 0;
1242  if(std::equal(buffer.end() - 6, buffer.end(), "define")) { // TODO: Maybe add support for
1243  // this? This would fill feature
1244  // request #21343
1245  parent_.error(
1246  "Preprocessor error: #define is not allowed inside a #define/#enddef pair",
1247  linenum);
1248  }
1249  }
1250  }
1251  }
1252  }
1253 
1254  if(found_enddef != 7) {
1255  parent_.error("Unterminated preprocessor definition", linenum_);
1256  }
1257 
1258  if(!skipping_) {
1259  preproc_map::const_iterator old_i = parent_.defines_->find(symbol);
1260  if(old_i != parent_.defines_->end()) {
1261  std::ostringstream new_pos, old_pos;
1262  const preproc_define& old_d = old_i->second;
1263 
1264  new_pos << linenum << ' ' << parent_.location_;
1265  old_pos << old_d.linenum << ' ' << old_d.location;
1266 
1267  WRN_PREPROC << "Redefining macro " << symbol << " without explicit #undef at "
1268  << lineno_string(new_pos.str()) << '\n'
1269  << "previously defined at " << lineno_string(old_pos.str()) << '\n';
1270  }
1271 
1272  buffer.erase(buffer.end() - 7, buffer.end());
1273  (*parent_.defines_)[symbol]
1274  = preproc_define(buffer, items, optargs, parent_.textdomain_, linenum, parent_.location_,
1275  deprecation_detail, deprecation_level, deprecation_version);
1276 
1277  LOG_PREPROC << "defining macro " << symbol << " (location " << get_location(parent_.location_) << ")\n";
1278  }
1279  } else if(command == "ifdef") {
1280  skip_spaces();
1281  const std::string& symbol = read_word();
1282  bool found = parent_.defines_->count(symbol) != 0;
1283  DBG_PREPROC << "testing for macro " << symbol << ": " << (found ? "defined" : "not defined") << '\n';
1284  conditional_skip(!found);
1285  } else if(command == "ifndef") {
1286  skip_spaces();
1287  const std::string& symbol = read_word();
1288  bool found = parent_.defines_->count(symbol) != 0;
1289  DBG_PREPROC << "testing for macro " << symbol << ": " << (found ? "defined" : "not defined") << '\n';
1290  conditional_skip(found);
1291  } else if(command == "ifhave") {
1292  skip_spaces();
1293  const std::string& symbol = read_word();
1294  bool found = !filesystem::get_wml_location(symbol, directory_).empty();
1295  DBG_PREPROC << "testing for file or directory " << symbol << ": " << (found ? "found" : "not found")
1296  << '\n';
1297  conditional_skip(!found);
1298  } else if(command == "ifnhave") {
1299  skip_spaces();
1300  const std::string& symbol = read_word();
1301  bool found = !filesystem::get_wml_location(symbol, directory_).empty();
1302  DBG_PREPROC << "testing for file or directory " << symbol << ": " << (found ? "found" : "not found")
1303  << '\n';
1304  conditional_skip(found);
1305  } else if(command == "ifver" || command == "ifnver") {
1306  skip_spaces();
1307  const std::string& vsymstr = read_word();
1308  skip_spaces();
1309  const std::string& vopstr = read_word();
1310  skip_spaces();
1311  const std::string& vverstr = read_word();
1312 
1313  const VERSION_COMP_OP vop = parse_version_op(vopstr);
1314 
1315  if(vop == OP_INVALID) {
1316  parent_.error("Invalid #ifver/#ifnver operator", linenum_);
1317  } else if(parent_.defines_->count(vsymstr) != 0) {
1318  const preproc_define& sym = (*parent_.defines_)[vsymstr];
1319 
1320  if(!sym.arguments.empty()) {
1321  parent_.error("First argument macro in #ifver/#ifnver should not require arguments", linenum_);
1322  }
1323 
1324  version_info const version1(sym.value);
1325  version_info const version2(vverstr);
1326 
1327  const bool found = do_version_check(version1, vop, version2);
1328  DBG_PREPROC << "testing version '" << version1.str() << "' against '" << version2.str() << "' ("
1329  << vopstr << "): " << (found ? "match" : "no match") << '\n';
1330 
1331  conditional_skip(command == "ifver" ? !found : found);
1332  } else {
1333  std::string err = "Undefined macro in #ifver/#ifnver first argument: '";
1334  err += vsymstr;
1335  err += "'";
1336  parent_.error(err, linenum_);
1337  }
1338  } else if(command == "else") {
1339  if(token.type == token_desc::SKIP_ELSE) {
1340  pop_token();
1341  --skipping_;
1343  } else if(token.type == token_desc::PROCESS_IF) {
1344  pop_token();
1345  ++skipping_;
1347  } else {
1348  parent_.error("Unexpected #else", linenum_);
1349  }
1350  } else if(command == "endif") {
1351  switch(token.type) {
1352  case token_desc::SKIP_IF:
1353  case token_desc::SKIP_ELSE:
1354  --skipping_;
1357  break;
1358  default:
1359  parent_.error("Unexpected #endif", linenum_);
1360  }
1361  pop_token();
1362  } else if(command == "textdomain") {
1363  skip_spaces();
1364  const std::string& s = read_word();
1365  if(s != parent_.textdomain_) {
1366  put("#textdomain ");
1367  put(s);
1368  parent_.textdomain_ = s;
1369  }
1370  comment = true;
1371  } else if(command == "enddef") {
1372  parent_.error("Unexpected #enddef", linenum_);
1373  } else if(command == "undef") {
1374  skip_spaces();
1375  const std::string& symbol = read_word();
1376  if(!skipping_) {
1377  parent_.defines_->erase(symbol);
1378  LOG_PREPROC << "undefine macro " << symbol << " (location " << get_location(parent_.location_) << ")\n";
1379  }
1380  } else if(command == "error") {
1381  if(!skipping_) {
1382  skip_spaces();
1383  std::ostringstream error;
1384  error << "#error: \"" << read_rest_of_line() << '"';
1385  parent_.error(error.str(), linenum_);
1386  } else
1387  DBG_PREPROC << "Skipped an error\n";
1388  } else if(command == "warning") {
1389  if(!skipping_) {
1390  skip_spaces();
1391  std::ostringstream warning;
1392  warning << "#warning: \"" << read_rest_of_line() << '"';
1393  parent_.warning(warning.str(), linenum_);
1394  } else {
1395  DBG_PREPROC << "Skipped a warning\n";
1396  }
1397  } else if(command == "deprecated") {
1398  // The current file is deprecated, so print a message
1399  skip_spaces();
1400  int level = 0;
1401  try {
1402  level = std::stoi(read_word());
1403  } catch(std::invalid_argument&) {}
1405  if(level == 2 || level == 3) {
1406  skip_spaces();
1407  version = version_info(read_word());
1408  }
1409  skip_spaces();
1411  deprecated_message(get_filename(parent_.location_), level, version, detail);
1412  } else {
1413  comment = token.type != token_desc::MACRO_SPACE;
1414  }
1415 
1416  skip_eol();
1417  if(comment) {
1418  put('\n');
1419  }
1420  } else if(token.type == token_desc::MACRO_SPACE || token.type == token_desc::MACRO_CHUNK) {
1421  if(c == '(') {
1422  // If a macro argument was started, it is implicitly ended.
1423  token.type = token_desc::MACRO_SPACE;
1425  } else if(utils::portable_isspace(c)) {
1426  // If a macro argument was started, it is implicitly ended.
1427  token.type = token_desc::MACRO_SPACE;
1428  } else if(c == '}') {
1429  --slowpath_;
1430  if(skipping_) {
1431  pop_token();
1432  return true;
1433  }
1434 
1435  // FIXME: is this obsolete?
1436  // if (token.type == token_desc::MACRO_SPACE) {
1437  // if (!strings_.back().empty()) {
1438  // std::ostringstream error;
1439  // std::ostringstream location;
1440  // error << "Can't parse new macro parameter with a macro call scope open";
1441  // location<<linenum_<<' '<<parent_.location_;
1442  // parent_.error(error.str(), location.str());
1443  // }
1444  // strings_.pop_back();
1445  //}
1446 
1447  if(strings_.size() <= static_cast<size_t>(token.stack_pos)) {
1448  parent_.error("No macro or file substitution target specified", linenum_);
1449  }
1450 
1451  std::string symbol = strings_[token.stack_pos];
1452  std::string::size_type pos;
1453  while((pos = symbol.find(OUTPUT_SEPARATOR)) != std::string::npos) {
1454  std::string::iterator b = symbol.begin(); // invalidated at each iteration
1455  symbol.erase(b + pos, b + symbol.find('\n', pos + 1) + 1);
1456  }
1457 
1458  std::map<std::string, std::string>::const_iterator arg;
1459  preproc_map::const_iterator macro;
1460 
1461  // If this is a known pre-processing symbol, then we insert it,
1462  // otherwise we assume it's a file name to load.
1463  if(symbol == current_file_str && strings_.size() - token.stack_pos == 1) {
1464  pop_token();
1466  } else if(symbol == current_dir_str && strings_.size() - token.stack_pos == 1) {
1467  pop_token();
1469  } else if(local_defines_ && (arg = local_defines_->find(symbol)) != local_defines_->end()) {
1470  if(strings_.size() - token.stack_pos != 1) {
1471  std::ostringstream error;
1472  error << "Macro argument '" << symbol << "' does not expect any arguments";
1473  parent_.error(error.str(), linenum_);
1474  }
1475 
1476  std::ostringstream v;
1477  v << arg->second << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1478  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1479 
1480  pop_token();
1481  put(v.str());
1482  } else if(parent_.depth() < 100 && (macro = parent_.defines_->find(symbol)) != parent_.defines_->end()) {
1483  const preproc_define& val = macro->second;
1484  size_t nb_arg = strings_.size() - token.stack_pos - 1;
1485  size_t optional_arg_num = 0;
1486 
1487  std::unique_ptr<std::map<std::string, std::string>> defines{new std::map<std::string, std::string>};
1488  const std::string& dir = filesystem::directory_name(val.location.substr(0, val.location.find(' ')));
1489 
1490  if(val.is_deprecated()) {
1492  }
1493 
1494  for(size_t i = 0; i < nb_arg; ++i) {
1495  if(i < val.arguments.size()) {
1496  // Normal mandatory arguments
1497 
1498  (*defines)[val.arguments[i]] = strings_[token.stack_pos + i + 1];
1499  } else {
1500  // These should be optional argument overrides
1501 
1502  std::string str = strings_[token.stack_pos + i + 1];
1503  size_t equals_pos = str.find_first_of("=");
1504 
1505  if(equals_pos != std::string::npos) {
1506  size_t argname_pos = str.substr(0, equals_pos).find_last_of(" \n") + 1;
1507 
1508  std::string argname = str.substr(argname_pos, equals_pos - argname_pos);
1509 
1510  if(val.optional_arguments.find(argname) != val.optional_arguments.end()) {
1511  (*defines)[argname] = str.substr(equals_pos + 1);
1512 
1513  optional_arg_num++;
1514 
1515  DBG_PREPROC << "Found override for " << argname << " in call to macro " << symbol
1516  << "\n";
1517  } else {
1518  std::ostringstream warning;
1519  warning << "Unrecognized optional argument passed to macro '" << symbol << "': '"
1520  << argname << "'";
1521  parent_.warning(warning.str(), linenum_);
1522 
1523  optional_arg_num++; // To prevent the argument number check from blowing up
1524  }
1525  }
1526  }
1527  }
1528 
1529  // If the macro definition has any optional arguments, insert their defaults
1530  if(val.optional_arguments.size() > 0) {
1531  for(const auto& argument : val.optional_arguments) {
1532  if(defines->find(argument.first) == defines->end()) {
1533  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1534 
1536  std::istream in(buf.get());
1537 
1538  filesystem::scoped_istream buffer{new std::istringstream(argument.second)};
1539 
1540  std::unique_ptr<std::map<std::string, std::string>> temp_defines{new std::map<std::string, std::string>};
1541  temp_defines->insert(defines->begin(), defines->end());
1542 
1544  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(temp_defines), false);
1545 
1546  std::ostringstream res;
1547  res << in.rdbuf();
1548 
1549  DBG_PREPROC << "Setting default for optional argument " << argument.first << " in macro "
1550  << symbol << "\n";
1551 
1552  (*defines)[argument.first] = res.str();
1553  }
1554  }
1555  }
1556 
1557  if(nb_arg - optional_arg_num != val.arguments.size()) {
1558  const std::vector<std::string>& locations = utils::quoted_split(val.location, ' ');
1559  std::ostringstream error;
1560  error << "Preprocessor symbol '" << symbol << "' defined at " << get_filename(locations[0]) << ":"
1561  << val.linenum << " expects " << val.arguments.size() << " arguments, but has "
1562  << nb_arg - optional_arg_num << " arguments";
1563  parent_.error(error.str(), linenum_);
1564  }
1565 
1566  filesystem::scoped_istream buffer{new std::istringstream(val.value)};
1567 
1568  pop_token();
1569 
1570  if(!slowpath_) {
1571  DBG_PREPROC << "substituting macro " << symbol << '\n';
1572 
1574  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1575  } else {
1576  DBG_PREPROC << "substituting (slow) macro " << symbol << '\n';
1577 
1578  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1579 
1580  // Make the nested preprocessor_data responsible for
1581  // restoring our current textdomain if needed.
1583 
1584  std::ostringstream res;
1585  {
1586  std::istream in(buf.get());
1588  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1589 
1590  res << in.rdbuf();
1591  }
1592 
1593  put(res.str());
1594  }
1595  } else if(parent_.depth() < 40) {
1596  LOG_PREPROC << "Macro definition not found for " << symbol << " , attempting to open as file.\n";
1597  pop_token();
1598 
1600  if(!nfname.empty()) {
1601  if(!slowpath_)
1602  // nfname.size() - symbol.size() gives you an index into nfname
1603  // This does not necessarily match the symbol though, as it can start with ~ or ./
1604  parent_.add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1605  else {
1606  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1607 
1608  std::ostringstream res;
1609  {
1610  std::istream in(buf.get());
1611  buf->add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1612 
1613  res << in.rdbuf();
1614  }
1615 
1616  put(res.str());
1617  }
1618  } else {
1619  std::ostringstream error;
1620  error << "Macro/file '" << symbol << "' is missing";
1621  parent_.error(error.str(), linenum_);
1622  }
1623  } else {
1624  parent_.error("Too many nested preprocessing inclusions", linenum_);
1625  }
1626  } else if(!skipping_) {
1627  if(token.type == token_desc::MACRO_SPACE) {
1628  std::ostringstream s;
1629  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1630  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1631 
1632  strings_.push_back(s.str());
1633  token.type = token_desc::MACRO_CHUNK;
1634  }
1635  put(c);
1636  }
1637  } else {
1638  put(c);
1639  }
1640 
1641  return true;
1642 }
1643 
1644 
1645 // ==================================================================================
1646 // PREPROCESSOR SCOPE HELPER
1647 // ==================================================================================
1648 
1649 struct preprocessor_scope_helper : std::basic_istream<char>
1650 {
1652  : std::basic_istream<char>(nullptr)
1653  , buf_(nullptr)
1654  , local_defines_(nullptr)
1655  {
1656  //
1657  // If no defines were provided, we create a new local preproc_map and assign
1658  // it to defines temporarily. In this case, the map will be deleted once this
1659  // object is destroyed and defines will still be subsequently null.
1660  //
1661  if(!defines) {
1662  local_defines_.reset(new preproc_map);
1663  defines = local_defines_.get();
1664  }
1665 
1666  buf_.reset(new preprocessor_streambuf(defines));
1667 
1668  // Begin processing.
1669  buf_->add_preprocessor<preprocessor_file>(fname);
1670 
1671  //
1672  // TODO: not sure if this call is needed. Previously, this call was passed a
1673  // preprocessor_streambuf pointer and the std::basic_istream constructor was
1674  // called with its contents. However, at that point the preprocessing should
1675  // already have completed, meaning this call might be redundant. Not sure.
1676  //
1677  // - vultraz, 2017-08-31
1678  //
1679  init(buf_.get());
1680  }
1681 
1683  {
1684  clear(std::ios_base::goodbit);
1685  exceptions(std::ios_base::goodbit);
1686  rdbuf(nullptr);
1687  }
1688 
1689  std::unique_ptr<preprocessor_streambuf> buf_;
1690  std::unique_ptr<preproc_map> local_defines_;
1691 };
1692 
1693 
1694 // ==================================================================================
1695 // FREE-STANDING FUNCTIONS
1696 // ==================================================================================
1697 
1699 {
1700  log_scope("preprocessing file " + fname + " ...");
1701 
1702  // NOTE: the preprocessor_scope_helper does *not* take ownership of defines.
1703  return filesystem::scoped_istream(new preprocessor_scope_helper(fname, defines));
1704 }
1705 
1706 void preprocess_resource(const std::string& res_name,
1707  preproc_map* defines_map,
1708  bool write_cfg,
1709  bool write_plain_cfg,
1710  const std::string& parent_directory)
1711 {
1712  if(filesystem::is_directory(res_name)) {
1713  std::vector<std::string> dirs, files;
1714 
1717 
1718  // Subdirectories
1719  for(const std::string& dir : dirs) {
1720  LOG_PREPROC << "processing sub-dir: " << dir << '\n';
1721  preprocess_resource(dir, defines_map, write_cfg, write_plain_cfg, parent_directory);
1722  }
1723 
1724  // Files in current directory
1725  for(const std::string& file : files) {
1726  preprocess_resource(file, defines_map, write_cfg, write_plain_cfg, parent_directory);
1727  }
1728 
1729  return;
1730  }
1731 
1732  // process only config files.
1733  if(!filesystem::ends_with(res_name, ".cfg")) {
1734  return;
1735  }
1736 
1737  LOG_PREPROC << "processing resource: " << res_name << '\n';
1738 
1739  // disable filename encoding to get clear #line in cfg.plain
1740  encode_filename = false;
1741 
1742  filesystem::scoped_istream stream = preprocess_file(res_name, defines_map);
1743 
1744  std::stringstream ss;
1745 
1746  // Set the failbit so if we get any preprocessor exceptions (e.g.:preproc_config::error)
1747  // they will be propagated in the main program, instead of just setting the
1748  // failbit on the stream. This was necessary in order for the MSVC and GCC
1749  // binaries to behave the same way.
1750  ss.exceptions(std::ios_base::failbit);
1751 
1752  ss << (*stream).rdbuf();
1753 
1754  LOG_PREPROC << "processing finished\n";
1755 
1756  if(write_cfg || write_plain_cfg) {
1757  config cfg;
1758  std::string streamContent = ss.str();
1759 
1760  read(cfg, streamContent);
1761 
1762  const std::string preproc_res_name = parent_directory + "/" + filesystem::base_name(res_name);
1763 
1764  // Write the processed cfg file
1765  if(write_cfg) {
1766  LOG_PREPROC << "writing cfg file: " << preproc_res_name << '\n';
1767 
1769  filesystem::scoped_ostream outStream(filesystem::ostream_file(preproc_res_name));
1770 
1771  write(*outStream, cfg);
1772  }
1773 
1774  // Write the plain cfg file
1775  if(write_plain_cfg) {
1776  LOG_PREPROC << "writing plain cfg file: " << (preproc_res_name + ".plain") << '\n';
1777 
1779  filesystem::write_file(preproc_res_name + ".plain", streamContent);
1780  }
1781  }
1782 }
static std::string current_file_str
std::string lineno_string(const std::string &lineno)
static const char OUTPUT_SEPARATOR
std::vector< std::string > arguments
std::vector< char_t > string
std::vector< std::string > files_
virtual bool get_chunk() override
Inserts and processes the next file in the list of included files.
VERSION_COMP_OP parse_version_op(const std::string &op_str)
Definition: version.cpp:279
preprocessor_scope_helper(const std::string &fname, preproc_map *defines)
Specialized preprocessor for handling a file or a set of files.
void read(const config &)
std::vector< std::string >::const_iterator end_
std::string read_word()
preproc_map default_defines_
static l_noret error(LoadState *S, const char *why)
Definition: lundump.cpp:39
buffered_istream in_
Input stream.
static std::map< std::string, int > file_number_map
bool ends_with(const std::string &str, const std::string &suffix)
#define ERR_PREPROC
child_itors child_range(config_key_type key)
Definition: config.cpp:362
virtual MODE parse_mode()
Returns the appropriate parsing mode for this preprocessor.
int peek()
Gets a character from the buffer.
Specialized preprocessor for handling any kind of input stream.
std::string out_buffer_
Buffer read by the STL stream.
void clear(const std::string &key)
Definition: general.cpp:207
Helper class for buffering a std::istream.
void write_argument(config_writer &, const std::string &) const
virtual bool get_chunk() override
Preprocesses and sends some text to the parent_ buffer.
std::vector< std::string > quoted_split(const std::string &val, char c, int flags, char quote)
This function is identical to split(), except it does not split when it otherwise would if the previo...
bool eof() const
Is the end of input reached?
void preprocess_resource(const std::string &res_name, preproc_map *defines_map, bool write_cfg, bool write_plain_cfg, const std::string &parent_directory)
STL namespace.
void read_argument(const config &)
const std::vector< std::string > items
#define d
std::unique_ptr< preprocessor_streambuf > buf_
Definitions for the interface to Wesnoth Markup Language (WML).
std::vector< std::string > split(const std::string &val, const char c, const int flags)
Splits a (comma-)separated string into a vector of pieces.
std::vector< std::string >::const_iterator pos_
std::string location
Description of a preprocessing chunk.
#define WRN_PREPROC
#define b
unsigned in
If equal to search_counter, the node is off the list.
preprocessor * current() const
void write(std::ostream &out, const configr_of &cfg, unsigned int level)
Definition: parser.cpp:749
int slowpath_
Set to true whenever input tokens cannot be directly sent to the target buffer.
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error=true)
bool is_define_
True iff we are currently parsing a macros content, otherwise false.
void write_key_val(const std::string &key, const T &value)
This template function will work with any type that can be assigned to an attribute_value.
struct utils::detail::formula_initer init
int get()
Gets and consumes a character from the buffer.
void conditional_skip(bool skip)
bool quoted_
Set to true if one preprocessor for this target started to read a string.
void close_child(const std::string &key)
bool create_directory_if_missing_recursive(const std::string &dirname)
Creates a recursive directory tree if it does not exist already.
static lg::log_domain log_preprocessor("preprocessor")
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:612
Class for writing a config out to a file in pieces.
version_info deprecation_version
void restore_old_preprocessor()
Restores the old preprocessing context.
std::string read_line()
void write_file(const std::string &fname, const std::string &data)
Throws io_exception if an error occurs.
int skipping_
Non-zero when the preprocessor has to skip some input text.
void push_token(token_desc::TOKEN_TYPE)
void open_child(const std::string &key)
std::deque< std::unique_ptr< preprocessor > > preprocessor_queue_
Input preprocessor queue.
virtual ~preprocessor()
bool is_directory(const std::string &fname)
Returns true if the given file is a directory.
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:37
void add_preprocessor(A &&...args)
virtual void init() override
Allows specifying any actions that need to be called after the constructor completes.
std::string get_short_wml_path(const std::string &filename)
Returns a short path to filename, skipping the (user) data directory.
std::string deprecation_message
#define DBG_PREPROC
std::string value
#define LOG_PREPROC
Instruction that causes us to skip upcoming instructions.
std::string str() const
Serializes the version number into string form.
Definition: version.cpp:93
std::string escape(const std::string &str, const char *special_chars)
Prepends a configurable set of characters with a backslash.
#define PACKAGE
Definition: wesconfig.h:23
std::string directory_
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:38
virtual preprocessor::MODE parse_mode() override
Returns the appropriate parsing mode for this preprocessor.
static std::string get_file_code(const std::string &filename)
Some defines: VERSION, PACKAGE, MIN_SAVEGAME_VERSION.
bool is_deprecated() const
virtual void init()
Allows specifying any actions that need to be called after the constructor completes.
filesystem::scoped_ostream ostream_file(const std::string &fname, bool create_directory=true)
token_desc(TOKEN_TYPE type, const int stack_pos, const int linenum)
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs=nullptr, file_name_option mode=FILE_NAME_ONLY, file_filter_option filter=NO_FILTER, file_reorder_option reorder=DONT_REORDER, file_tree_checksum *checksum=nullptr)
Populates 'files' with all the files and 'dirs' with all the directories in dir.
VERSION_COMP_OP
Definition: version.hpp:194
logger & err()
Definition: log.cpp:79
std::string get_wml_location(const std::string &filename, const std::string &current_dir=std::string())
Returns a complete path to the actual WML file or directory or an empty string if the file isn't pres...
std::string read_rest_of_line()
std::unique_ptr< preproc_map > local_defines_
bool operator!=(preprocessor_data::token_desc::TOKEN_TYPE rhs, char lhs)
mock_party p
static std::string get_location(const std::string &loc)
static map_location::DIRECTION s
filesystem::scoped_istream in_scope_
Manages the lifetime of the std::istream pointer we own.
friend class preprocessor_streambuf
#define log_scope(description)
Definition: log.hpp:186
size_t i
Definition: function.cpp:933
void write(config_writer &, const std::string &) const
Helper class for buffering a std::istream.
std::unique_ptr< std::map< std::string, std::string > > local_defines_
Mapping of macro arguments to their content.
Declarations for File-IO.
static int writer(lua_State *L, const void *b, size_t size, void *B)
Definition: lstrlib.cpp:182
void warning(const std::string &, int)
const version_info wesnoth_version(VERSION)
Base class for preprocessing an input.
std::vector< std::string > strings_
Buffer for delayed input processing.
Represents version numbers.
Definition: version.hpp:43
std::ostream & operator<<(std::ostream &stream, const preproc_define &def)
static std::string current_dir_str
std::string base_name(const std::string &file, const bool remove_extension=false)
Returns the base filename of a file, with directory name stripped.
preprocessor_streambuf(const preprocessor_streambuf &t)
static std::string get_filename(const std::string &file_code)
double t
Definition: astarsearch.cpp:64
std::vector< token_desc > tokens_
Stack of nested preprocessing chunks.
preprocessor_file(preprocessor_streambuf &t, const std::string &name, size_t symbol_index=-1)
Constructor.
preprocessor_streambuf & parent_
Standard logging facilities (interface).
std::string textdomain
virtual int underflow() override
Inherited from basic_streambuf.
virtual bool get_chunk()=0
Preprocesses and sends some text to the parent_ buffer.
int stack_pos
Starting position in strings_ of the delayed text for this chunk.
preprocessor_data(preprocessor_streambuf &, filesystem::scoped_istream, const std::string &history, const std::string &name, int line, const std::string &dir, const std::string &domain, std::unique_ptr< std::map< std::string, std::string >> defines, bool is_define=false)
static const char * name(const std::vector< SDL_Joystick * > &joysticks, const size_t index)
Definition: joystick.cpp:48
friend bool operator==(preprocessor_data::token_desc::TOKEN_TYPE, char)
void error(const std::string &, int)
Target for sending preprocessed output.
std::map< std::string, struct preproc_define > preproc_map
#define e
bool do_version_check(const version_info &a, VERSION_COMP_OP op, const version_info &b)
Definition: version.cpp:298
bool portable_isspace(const char c)
preprocessor_streambuf(preproc_map *def)
bool operator==(preprocessor_data::token_desc::TOKEN_TYPE, char)
std::string get_current_file()
Decodes the filenames placed in a location.
std::string old_location_
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:93
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
static preproc_map::value_type read_pair(const config &)
mock_char c
const std::string & name_
std::string old_textdomain_
Interfaces for manipulating version numbers of engine, add-ons, etc.
static map_location::DIRECTION n
void deprecated_message(const std::string &, int, const version_info &, const std::string &)
const std::string version
Definition: game_config.cpp:39
static bool encode_filename
std::string::const_iterator iterator
Definition: tokenizer.hpp:24
bool operator<(const preproc_define &) const
std::stringstream buffer_
Buffer filled by the #current_ preprocessor.
preprocessor(preprocessor_streambuf &t)
Sets up a new preprocessor for stream buffer t.
std::string directory_name(const std::string &file)
Returns the directory name of a file, with filename stripped.
bool operator==(const preproc_define &) const
friend bool operator!=(preprocessor_data::token_desc::TOKEN_TYPE, char)
static std::string preprocessor_error_detail_prefix
std::map< std::string, std::string > optional_arguments