The Battle for Wesnoth  1.15.0-dev
preprocessor.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2003 by David White <dave@whitevine.net>
3  Copyright (C) 2005 - 2018 by Guillaume Melquiond <guillaume.melquiond@gmail.com>
4  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation; either version 2 of the License, or
9  (at your option) any later version.
10  This program is distributed in the hope that it will be useful,
11  but WITHOUT ANY WARRANTY.
12 
13  See the COPYING file for more details.
14 */
15 
16 /**
17  * @file
18  * WML preprocessor.
19  */
20 
22 
23 #include "buffered_istream.hpp"
24 #include "config.hpp"
25 #include "filesystem.hpp"
26 #include "log.hpp"
28 #include "serialization/parser.hpp"
30 #include "game_version.hpp"
31 #include "wesconfig.h"
32 #include "deprecation.hpp"
33 
34 #include <stdexcept>
35 #include <deque>
36 
37 static lg::log_domain log_preprocessor("preprocessor");
38 #define ERR_PREPROC LOG_STREAM(err, log_preprocessor)
39 #define WRN_PREPROC LOG_STREAM(warn, log_preprocessor)
40 #define LOG_PREPROC LOG_STREAM(info, log_preprocessor)
41 #define DBG_PREPROC LOG_STREAM(debug, log_preprocessor)
42 
43 static std::string current_file_str = "CURRENT_FILE";
44 static std::string current_dir_str = "CURRENT_DIRECTORY";
45 
46 // map associating each filename encountered to a number
47 static std::map<std::string, int> file_number_map;
48 
49 static bool encode_filename = true;
50 
51 static std::string preprocessor_error_detail_prefix = "\n ";
52 
53 static const char OUTPUT_SEPARATOR = '\xFE';
54 
55 // get filename associated to this code
56 static std::string get_filename(const std::string& file_code)
57 {
58  if(!encode_filename) {
59  return file_code;
60  }
61 
62  std::stringstream s;
63  s << file_code;
64  int n = 0;
65  s >> std::hex >> n;
66 
67  for(const auto& p : file_number_map) {
68  if(p.second == n) {
69  return p.first;
70  }
71  }
72 
73  return "<unknown>";
74 }
75 
76 // Get code associated to this filename
77 static std::string get_file_code(const std::string& filename)
78 {
79  if(!encode_filename) {
80  return filename;
81  }
82 
83  // Current number of encountered filenames
84  static int current_file_number = 0;
85 
86  int& fnum = file_number_map[utils::escape(filename, " \\")];
87  if(fnum == 0) {
88  fnum = ++current_file_number;
89  }
90 
91  std::ostringstream shex;
92  shex << std::hex << fnum;
93 
94  return shex.str();
95 }
96 
97 // decode the filenames placed in a location
98 static std::string get_location(const std::string& loc)
99 {
100  std::string res;
101  std::vector<std::string> pos = utils::quoted_split(loc, ' ');
102 
103  if(pos.empty()) {
104  return res;
105  }
106 
107  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
108  while(true) {
109  res += get_filename(*(i++));
110 
111  if(i == end) {
112  break;
113  }
114 
115  res += ' ';
116  res += *(i++);
117 
118  if(i == end) {
119  break;
120  }
121 
122  res += ' ';
123  }
124 
125  return res;
126 }
127 
128 
129 // ==================================================================================
130 // PREPROC_DEFINE IMPLEMENTATION
131 // ==================================================================================
132 
134 {
135  return value == v.value && arguments == v.arguments;
136 }
137 
139 {
140  if(location < v.location) {
141  return true;
142  }
143 
144  if(v.location < location) {
145  return false;
146  }
147 
148  return linenum < v.linenum;
149 }
150 
151 void preproc_define::write_argument(config_writer& writer, const std::string& arg) const
152 {
153  const std::string key = "argument";
154 
155  writer.open_child(key);
156 
157  writer.write_key_val("name", arg);
158  writer.close_child(key);
159 }
160 
161 void preproc_define::write(config_writer& writer, const std::string& name) const
162 {
163  const std::string key = "preproc_define";
164  writer.open_child(key);
165 
166  writer.write_key_val("name", name);
167  writer.write_key_val("value", value);
168  writer.write_key_val("textdomain", textdomain);
169  writer.write_key_val("linenum", std::to_string(linenum));
170  writer.write_key_val("location", get_location(location));
171 
172  for(const std::string& arg : arguments) {
173  write_argument(writer, arg);
174  }
175 
176  writer.close_child(key);
177 }
178 
180 {
181  arguments.push_back(cfg["name"]);
182 }
183 
184 void preproc_define::read(const config& cfg)
185 {
186  value = cfg["value"].str();
187  textdomain = cfg["textdomain"].str();
188  linenum = cfg["linenum"];
189  location = cfg["location"].str();
190 
191  for(const config& arg : cfg.child_range("argument")) {
192  read_argument(arg);
193  }
194 }
195 
196 preproc_map::value_type preproc_define::read_pair(const config& cfg)
197 {
198  preproc_define second;
199  second.read(cfg);
200 
201  return preproc_map::value_type(cfg["name"], second);
202 }
203 
204 std::ostream& operator<<(std::ostream& stream, const preproc_define& def)
205 {
206  return stream << "value: " << def.value << " arguments: " << def.location;
207 }
208 
209 std::ostream& operator<<(std::ostream& stream, const preproc_map::value_type& def)
210 {
211  return stream << def.second;
212 }
213 
214 // ==================================================================================
215 // PREPROCESSOR BASE
216 // ==================================================================================
217 
219 
220 /**
221  * Base class for preprocessing an input.
222  */
224 {
226 
227 protected:
228  /**
229  * Sets up a new preprocessor for stream buffer \a t.
230  * Saves the current preprocessing context of #parent_. It will be automatically restored on destruction.
231  *
232  * It relies on preprocessor_streambuf so it's implemented after that class is declared.
233  */
235 
237 
238 public:
239  virtual ~preprocessor()
240  {
241  }
242 
243  /** Allows specifying any actions that need to be called after the constructor completes. */
244  virtual void init()
245  {
246  }
247 
248  /**
249  * Preprocesses and sends some text to the #parent_ buffer.
250  * @return false when the input has no data left.
251  */
252  virtual bool get_chunk() = 0;
253 
254  enum MODE { NO_PARSING, PARSES_FILE, PARSES_MACRO };
255 
256  /** Returns the appropriate parsing mode for this preprocessor. */
257  virtual MODE parse_mode()
258  {
259  return NO_PARSING;
260  }
261 
262 private:
263  std::string old_textdomain_;
264  std::string old_location_;
265 
267 };
268 
269 
270 // ==================================================================================
271 // PREPROCESSOR BUFFER
272 // ==================================================================================
273 
274 /**
275  * Target for sending preprocessed output.
276  * Objects of this class can be plugged into an STL stream.
277  */
278 class preprocessor_streambuf : public std::streambuf
279 {
280 public:
282  : std::streambuf()
283  , out_buffer_("")
284  , buffer_()
285  , preprocessor_queue_()
286  , defines_(def)
287  , default_defines_()
288  , textdomain_(PACKAGE)
289  , location_("")
290  , linenum_(0)
291  , quoted_(false)
292  {
293  }
294 
295  /** Decodes the filenames placed in a location. */
296  std::string get_current_file();
297 
298  void error(const std::string&, int);
299  void warning(const std::string&, int);
300 
301  template<typename T, typename... A>
302  void add_preprocessor(A&&... args)
303  {
304  preprocessor_queue_.emplace_back(new T(*this, std::forward<A>(args)...));
305  preprocessor_queue_.back()->init();
306  }
307 
309  {
310  preprocessor_queue_.pop_back();
311  }
312 
313  int depth() const
314  {
315  return preprocessor_queue_.size();
316  }
317 
319  {
320  return preprocessor_queue_.empty() ? nullptr : preprocessor_queue_.back().get();
321  }
322 
323 private:
325  : std::streambuf()
326  , out_buffer_("")
327  , buffer_()
328  , preprocessor_queue_()
329  , defines_(t.defines_)
330  , default_defines_()
331  , textdomain_(PACKAGE)
332  , location_("")
333  , linenum_(0)
334  , quoted_(t.quoted_)
335  {
336  }
337 
338  /** Inherited from basic_streambuf. */
339  virtual int underflow() override;
340 
341  void restore_old_preprocessor();
342 
343  /** Buffer read by the STL stream. */
344  std::string out_buffer_;
345 
346  /** Buffer filled by the #current_ preprocessor. */
347  std::stringstream buffer_;
348 
349  /** Input preprocessor queue. */
350  std::deque<std::unique_ptr<preprocessor>> preprocessor_queue_;
351 
354 
355  std::string textdomain_;
356  std::string location_;
357 
358  int linenum_;
359 
360  /**
361  * Set to true if one preprocessor for this target started to read a string.
362  * Deeper-nested preprocessors are then forbidden to.
363  */
364  bool quoted_;
365 
366  friend class preprocessor;
367  friend class preprocessor_file;
368  friend class preprocessor_data;
370 };
371 
372 /** Preprocessor constructor. */
374  : parent_(t)
375  , old_textdomain_(t.textdomain_)
376  , old_location_(t.location_)
377  , old_linenum_(t.linenum_)
378 {
379 }
380 
381 /**
382  * Called by an STL stream whenever it has reached the end of #out_buffer_.
383  * Fills #buffer_ by calling the #current_ preprocessor, then copies its
384  * content into #out_buffer_.
385  * @return the first character of #out_buffer_ if any, EOF otherwise.
386  */
388 {
389  unsigned sz = 0;
390  if(char* gp = gptr()) {
391  if(gp < egptr()) {
392  // Sanity check: the internal buffer has not been totally consumed,
393  // should we force the caller to use what remains first?
394  return *gp;
395  }
396 
397  // The buffer has been completely read; fill it again.
398  // Keep part of the previous buffer, to ensure putback capabilities.
399  sz = out_buffer_.size();
400  buffer_.str(std::string());
401 
402  if(sz > 3) {
403  buffer_ << out_buffer_.substr(sz - 3);
404  sz = 3;
405  } else {
406  buffer_ << out_buffer_;
407  }
408  } else {
409  // The internal get-data pointer is null
410  }
411 
412  const int desired_fill_amount = 2000;
413 
414  while(current() && buffer_.rdbuf()->in_avail() < desired_fill_amount) {
415  // Process files and data chunks until the desired buffer size is reached
416  if(!current()->get_chunk()) {
417  // Drop the current preprocessor item from the queue.
418  restore_old_preprocessor();
419  }
420  }
421 
422  // Update the internal state and data pointers
423  out_buffer_ = buffer_.str();
424  if(out_buffer_.empty()) {
425  return EOF;
426  }
427 
428  char* begin = &*out_buffer_.begin();
429  unsigned bs = out_buffer_.size();
430 
431  setg(begin, begin + sz, begin + bs);
432 
433  if(sz >= bs) {
434  return EOF;
435  }
436 
437  return static_cast<unsigned char>(*(begin + sz));
438 }
439 
440 /**
441 * Restores the old preprocessing context.
442 * Appends location and domain directives to the buffer, so that the parser
443 * notices these changes.
444 */
446 {
447  preprocessor* current = this->current();
448 
449  if(!current->old_location_.empty()) {
450  buffer_ << OUTPUT_SEPARATOR << "line " << current->old_linenum_ << ' ' << current->old_location_ << '\n';
451  }
452 
453  if(!current->old_textdomain_.empty() && textdomain_ != current->old_textdomain_) {
454  buffer_ << OUTPUT_SEPARATOR << "textdomain " << current->old_textdomain_ << '\n';
455  }
456 
457  location_ = current->old_location_;
458  linenum_ = current->old_linenum_;
459  textdomain_ = current->old_textdomain_;
460 
461  // Drop the preprocessor from the queue.
462  drop_preprocessor();
463 }
464 
466 {
467  unsigned nested_level = 0;
468 
469  preprocessor* pre = nullptr;
470 
471  // Iterate backwards over queue to get the last non-macro preprocessor.
472  for(auto p = preprocessor_queue_.rbegin(); p != preprocessor_queue_.rend(); ++p) {
473  pre = p->get();
474 
475  if(!pre || pre->parse_mode() == preprocessor::PARSES_FILE) {
476  break;
477  }
478 
479  if(pre->parse_mode() == preprocessor::PARSES_MACRO) {
480  ++nested_level;
481  }
482  }
483 
484  std::string res;
485  std::vector<std::string> pos = utils::quoted_split(location_, ' ');
486 
487  if(pos.size() <= 2 * nested_level) {
488  return res;
489  }
490 
491  return get_filename(pos[2 * nested_level]);
492 }
493 
494 std::string lineno_string(const std::string& lineno)
495 {
496  std::vector<std::string> pos = utils::quoted_split(lineno, ' ');
497  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
498  std::string included_from = preprocessor_error_detail_prefix + "included from ";
499  std::string res;
500 
501  while(i != end) {
502  const std::string& line = *(i++);
503 
504  if(!res.empty()) {
505  res += included_from;
506  }
507 
508  if(i != end) {
509  res += get_filename(*(i++));
510  } else {
511  res += "<unknown>";
512  }
513 
514  res += ':' + line;
515  }
516 
517  if(res.empty()) {
518  res = "???";
519  }
520 
521  return res;
522 }
523 
524 void preprocessor_streambuf::error(const std::string& error_type, int l)
525 {
526  std::string position, error;
527  std::ostringstream pos;
528 
529  pos << l << ' ' << location_;
530  position = lineno_string(pos.str());
531 
532  error = error_type + '\n';
533  error += "at " + position;
534 
535  ERR_PREPROC << error << '\n';
536 
537  throw preproc_config::error(error);
538 }
539 
540 void preprocessor_streambuf::warning(const std::string& warning_type, int l)
541 {
542  std::string position, warning;
543  std::ostringstream pos;
544 
545  pos << l << ' ' << location_;
546  position = lineno_string(pos.str());
547 
548  warning = warning_type + '\n';
549  warning += "at " + position;
550 
551  WRN_PREPROC << warning << '\n';
552 }
553 
554 
555 // ==================================================================================
556 // PREPROCESSOR FILE
557 // ==================================================================================
558 
559 /**
560  * Specialized preprocessor for handling a file or a set of files.
561  * A preprocessor_file object is created when a preprocessor encounters an
562  * inclusion directive that resolves to a file or directory, e.g. '{themes/}'.
563  */
565 {
566 public:
567  /** Constructor. It relies on preprocessor_data so it's implemented after that class is declared. */
568  preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index = -1);
569 
570  virtual void init() override;
571 
572  /**
573  * Inserts and processes the next file in the list of included files.
574  * @return false if there is no next file.
575  */
576  virtual bool get_chunk() override
577  {
578  while(pos_ != end_) {
579  const std::string& name = *(pos_++);
580  unsigned sz = name.size();
581 
582  // Use reverse iterator to optimize testing
583  if(sz < 5 || !std::equal(name.rbegin(), name.rbegin() + 4, "gfc.")) {
584  continue;
585  }
586 
588  return true;
589  }
590 
591  return false;
592  }
593 
594 private:
595  std::vector<std::string> files_;
596  std::vector<std::string>::const_iterator pos_, end_;
597 
598  const std::string& name_;
599 
601 };
602 
603 
604 // ==================================================================================
605 // PREPROCESSOR DATA
606 // ==================================================================================
607 
608 /**
609  * Specialized preprocessor for handling any kind of input stream.
610  * This is the core of the preprocessor.
611  */
613 {
614  /** Description of a preprocessing chunk. */
615  struct token_desc
616  {
617  enum TOKEN_TYPE {
618  START, // Toplevel
619  PROCESS_IF, // Processing the "if" branch of a ifdef/ifndef (the "else" branch will be skipped)
620  PROCESS_ELSE, // Processing the "else" branch of a ifdef/ifndef
621  SKIP_IF, // Skipping the "if" branch of a ifdef/ifndef (the "else" branch, if any, will be processed)
622  SKIP_ELSE, // Skipping the "else" branch of a ifdef/ifndef
623  STRING, // Processing a string
624  VERBATIM, // Processing a verbatim string
625  MACRO_SPACE, // Processing between chunks of a macro call (skip spaces)
626  MACRO_CHUNK, // Processing inside a chunk of a macro call (stop on space or '(')
627  MACRO_PARENS // Processing a parenthesized macro argument
628  };
629 
630  token_desc(TOKEN_TYPE type, const int stack_pos, const int linenum)
631  : type(type)
632  , stack_pos(stack_pos)
633  , linenum(linenum)
634  {
635  }
636 
638 
639  /** Starting position in #strings_ of the delayed text for this chunk. */
641  int linenum;
642  };
643 
644  /**
645  * Manages the lifetime of the @c std::istream pointer we own.
646  *
647  * Since @ref in_ uses the stream as well this object must be created
648  * before @ref in_ and destroyed after @ref in_ is destroyed.
649  */
651 
652  /** Input stream. */
654 
655  std::string directory_;
656 
657  /** Buffer for delayed input processing. */
658  std::vector<std::string> strings_;
659 
660  /** Mapping of macro arguments to their content. */
661  std::unique_ptr<std::map<std::string, std::string>> local_defines_;
662 
663  /** Stack of nested preprocessing chunks. */
664  std::vector<token_desc> tokens_;
665 
666  /**
667  * Set to true whenever input tokens cannot be directly sent to the target
668  * buffer. For instance, this happens with macro arguments. In that case,
669  * the output is redirected toward #strings_ until it can be processed.
670  */
672 
673  /**
674  * Non-zero when the preprocessor has to skip some input text.
675  * Increased whenever entering a conditional branch that is not useful,
676  * e.g. a ifdef that evaluates to false.
677  */
679  int linenum_;
680 
681  /** True iff we are currently parsing a macros content, otherwise false. */
683 
684  std::string read_word();
685  std::string read_line();
686  std::string read_rest_of_line();
687 
688  void skip_spaces();
689  void skip_eol();
690  void push_token(token_desc::TOKEN_TYPE);
691  void pop_token();
692  void put(char);
693  void put(const std::string& /*, int change_line = 0 */);
694  void conditional_skip(bool skip);
695 
696 public:
699  const std::string& history,
700  const std::string& name,
701  int line,
702  const std::string& dir,
703  const std::string& domain,
704  std::unique_ptr<std::map<std::string, std::string>> defines,
705  bool is_define = false);
706 
707  virtual bool get_chunk() override;
708 
709  virtual preprocessor::MODE parse_mode() override
710  {
711  return is_define_ ? PARSES_MACRO : PARSES_FILE;
712  }
713 
718 };
719 
721 {
722  throw std::logic_error("don't compare tokens with characters");
723 }
724 
726 {
727  return rhs == lhs;
728 }
729 
731 {
732  return !(lhs == rhs);
733 }
734 
736 {
737  return rhs != lhs;
738 }
739 
740 /** preprocessor_file constructor. */
741 preprocessor_file::preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index)
742  : preprocessor(t)
743  , files_()
744  , pos_()
745  , end_()
746  , name_(name)
747  , is_directory_(filesystem::is_directory(name))
748 {
749  if(is_directory_) {
750  filesystem::get_files_in_dir(name, &files_, nullptr,
754  );
755 
756  for(const std::string& fname : files_) {
757  std::size_t cpos = fname.rfind(" ");
758 
759  if(cpos != std::string::npos && cpos >= symbol_index) {
760  std::stringstream ss;
761  ss << "Found filename containing whitespace: '" << filesystem::base_name(fname)
762  << "' in included directory '" << name << "'.\nThe included symbol probably looks similar to '"
763  << filesystem::directory_name(fname.substr(symbol_index)) << "'";
764 
765  // TODO: find a real linenumber
766  parent_.error(ss.str(), -1);
767  }
768  }
769  } else {
770  // Handled in the init() function.
771  }
772 
773  pos_ = files_.begin();
774  end_ = files_.end();
775 }
776 
778 {
779  if(is_directory_) {
780  return;
781  }
782 
784 
785  if(!file_stream->good()) {
786  ERR_PREPROC << "Could not open file " << name_ << std::endl;
787  } else {
788  parent_.add_preprocessor<preprocessor_data>(std::move(file_stream), "",
791  );
792  }
793 }
794 
797  const std::string& history,
798  const std::string& name,
799  int linenum,
800  const std::string& directory,
801  const std::string& domain,
802  std::unique_ptr<std::map<std::string, std::string>> defines,
803  bool is_define)
804  : preprocessor(t)
805  , in_scope_(std::move(i))
806  , in_(*in_scope_)
807  , directory_(directory)
808  , strings_()
809  , local_defines_(std::move(defines))
810  , tokens_()
811  , slowpath_(0)
812  , skipping_(0)
813  , linenum_(linenum)
814  , is_define_(is_define)
815 {
816  std::ostringstream s;
817  s << history;
818 
819  if(!name.empty()) {
820  if(!history.empty()) {
821  s << ' ';
822  }
823 
824  s << get_file_code(name);
825  }
826 
827  if(!t.location_.empty()) {
828  s << ' ' << t.linenum_ << ' ' << t.location_;
829  }
830 
831  t.location_ = s.str();
832  t.linenum_ = linenum;
833 
834  t.buffer_ << OUTPUT_SEPARATOR << "line " << linenum << ' ' << t.location_ << '\n';
835 
836  if(t.textdomain_ != domain) {
837  t.buffer_ << OUTPUT_SEPARATOR << "textdomain " << domain << '\n';
838  t.textdomain_ = domain;
839  }
840 
842 }
843 
845 {
846  tokens_.emplace_back(t, strings_.size(), linenum_);
847 
848  if(t == token_desc::MACRO_SPACE) {
849  // Macro expansions do not have any associated storage at start.
850  return;
851  } else if(t == token_desc::STRING || t == token_desc::VERBATIM) {
852  /* Quoted strings are always inlined in the parent token. So
853  * they need neither storage nor metadata, unless the parent
854  * token is a macro expansion.
855  */
856  token_desc::TOKEN_TYPE& outer_type = tokens_[tokens_.size() - 2].type;
857  if(outer_type != token_desc::MACRO_SPACE) {
858  return;
859  }
860 
861  outer_type = token_desc::MACRO_CHUNK;
862  tokens_.back().stack_pos = strings_.size() + 1;
863  }
864 
865  std::ostringstream s;
866  if(!skipping_ && slowpath_) {
867  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
868  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
869  }
870 
871  strings_.push_back(s.str());
872 }
873 
875 {
876  token_desc::TOKEN_TYPE inner_type = tokens_.back().type;
877  unsigned stack_pos = tokens_.back().stack_pos;
878 
879  tokens_.pop_back();
880 
881  token_desc::TOKEN_TYPE& outer_type = tokens_.back().type;
882 
883  if(inner_type == token_desc::MACRO_PARENS) {
884  // Parenthesized macro arguments are left on the stack.
885  assert(outer_type == token_desc::MACRO_SPACE);
886  return;
887  }
888 
889  if(inner_type == token_desc::STRING || inner_type == token_desc::VERBATIM) {
890  // Quoted strings are always inlined.
891  assert(stack_pos == strings_.size());
892  return;
893  }
894 
895  if(outer_type == token_desc::MACRO_SPACE) {
896  /* A macro expansion does not have any associated storage.
897  * Instead, storage of the inner token is not discarded
898  * but kept as a new macro argument. But if the inner token
899  * was a macro expansion, it is about to be appended, so
900  * prepare for it.
901  */
902  if(inner_type == token_desc::MACRO_SPACE || inner_type == token_desc::MACRO_CHUNK) {
903  strings_.erase(strings_.begin() + stack_pos, strings_.end());
904  strings_.emplace_back();
905  }
906 
907  assert(stack_pos + 1 == strings_.size());
908  outer_type = token_desc::MACRO_CHUNK;
909 
910  return;
911  }
912 
913  strings_.erase(strings_.begin() + stack_pos, strings_.end());
914 }
915 
917 {
918  for(;;) {
919  int c = in_.peek();
920 
921  if(in_.eof() || (c != ' ' && c != '\t')) {
922  return;
923  }
924 
925  in_.get();
926  }
927 }
928 
930 {
931  for(;;) {
932  int c = in_.get();
933 
934  if(c == '\n') {
935  ++linenum_;
936  return;
937  }
938 
939  if(in_.eof()) {
940  return;
941  }
942  }
943 }
944 
946 {
947  std::string res;
948 
949  for(;;) {
950  int c = in_.peek();
951 
952  if(c == preprocessor_streambuf::traits_type::eof() || utils::portable_isspace(c)) {
953  // DBG_PREPROC << "(" << res << ")\n";
954  return res;
955  }
956 
957  in_.get();
958  res += static_cast<char>(c);
959  }
960 }
961 
963 {
964  std::string res;
965 
966  for(;;) {
967  int c = in_.get();
968 
969  if(c == '\n') {
970  ++linenum_;
971  return res;
972  }
973 
974  if(in_.eof()) {
975  return res;
976  }
977 
978  if(c != '\r') {
979  res += static_cast<char>(c);
980  }
981  }
982 }
983 
985 {
986  std::string res;
987 
988  while(in_.peek() != '\n' && !in_.eof()) {
989  int c = in_.get();
990 
991  if(c != '\r') {
992  res += static_cast<char>(c);
993  }
994  }
995 
996  return res;
997 }
998 
1000 {
1001  if(skipping_) {
1002  return;
1003  }
1004 
1005  if(slowpath_) {
1006  strings_.back() += c;
1007  return;
1008  }
1009 
1010  int cond_linenum = c == '\n' ? linenum_ - 1 : linenum_;
1011 
1012  if(unsigned diff = cond_linenum - parent_.linenum_) {
1013  parent_.linenum_ = cond_linenum;
1014 
1015  if(diff <= parent_.location_.size() + 11) {
1016  parent_.buffer_ << std::string(diff, '\n');
1017  } else {
1018  parent_.buffer_ << OUTPUT_SEPARATOR << "line " << parent_.linenum_ << ' ' << parent_.location_ << '\n';
1019  }
1020  }
1021 
1022  if(c == '\n') {
1023  ++parent_.linenum_;
1024  }
1025 
1026  parent_.buffer_ << c;
1027 }
1028 
1029 void preprocessor_data::put(const std::string& s /*, int line_change*/)
1030 {
1031  if(skipping_) {
1032  return;
1033  }
1034 
1035  if(slowpath_) {
1036  strings_.back() += s;
1037  return;
1038  }
1039 
1040  parent_.buffer_ << s;
1041  // parent_.linenum_ += line_change;
1042 }
1043 
1045 {
1046  if(skip) {
1047  ++skipping_;
1048  }
1049 
1051 }
1052 
1054 {
1055  char c = static_cast<char>(in_.get());
1056  token_desc& token = tokens_.back();
1057 
1058  if(in_.eof()) {
1059  // The end of file was reached.
1060  // Make sure we don't have any incomplete tokens.
1061  char const* s;
1062 
1063  switch(token.type) {
1064  case token_desc::START:
1065  return false; // everything is fine
1067  case token_desc::SKIP_IF:
1069  case token_desc::SKIP_ELSE:
1070  s = "#ifdef or #ifndef";
1071  break;
1072  case token_desc::STRING:
1073  s = "Quoted string";
1074  break;
1075  case token_desc::VERBATIM:
1076  s = "Verbatim string";
1077  break;
1080  s = "Macro substitution";
1081  break;
1083  s = "Macro argument";
1084  break;
1085  default:
1086  s = "???";
1087  }
1088 
1089  parent_.error(std::string(s) + " not terminated", token.linenum);
1090  }
1091 
1092  if(c == '\n') {
1093  ++linenum_;
1094  }
1095 
1096  if(c == OUTPUT_SEPARATOR) {
1097  std::string buffer(1, c);
1098 
1099  for(;;) {
1100  char d = static_cast<char>(in_.get());
1101 
1102  if(in_.eof() || d == '\n') {
1103  break;
1104  }
1105 
1106  buffer += d;
1107  }
1108 
1109  buffer += '\n';
1110  // line_change = 1-1 = 0
1111  put(buffer);
1112  } else if(token.type == token_desc::VERBATIM) {
1113  put(c);
1114 
1115  if(c == '>' && in_.peek() == '>') {
1116  put(in_.get());
1117  pop_token();
1118  }
1119  } else if(c == '<' && in_.peek() == '<') {
1120  in_.get();
1122  put('<');
1123  put('<');
1124  } else if(c == '"') {
1125  if(token.type == token_desc::STRING) {
1126  parent_.quoted_ = false;
1127  put(c);
1128  pop_token();
1129  } else if(!parent_.quoted_) {
1130  parent_.quoted_ = true;
1132  put(c);
1133  } else {
1134  parent_.error("Nested quoted string", linenum_);
1135  }
1136  } else if(c == '{') {
1138  ++slowpath_;
1139  } else if(c == ')' && token.type == token_desc::MACRO_PARENS) {
1140  pop_token();
1141  } else if(c == '#' && !parent_.quoted_) {
1142  std::string command = read_word();
1143  bool comment = false;
1144 
1145  if(command == "define") {
1146  skip_spaces();
1147  int linenum = linenum_;
1148  std::vector<std::string> items = utils::split(read_line(), ' ');
1149  std::map<std::string, std::string> optargs;
1150 
1151  if(items.empty()) {
1152  parent_.error("No macro name found after #define directive", linenum);
1153  }
1154 
1155  std::string symbol = items.front();
1156  items.erase(items.begin());
1157  int found_arg = 0, found_enddef = 0, found_deprecate = 0;
1158  boost::optional<DEP_LEVEL> deprecation_level = boost::none;
1159  std::string buffer, deprecation_detail;
1160  version_info deprecation_version = game_config::wesnoth_version;
1161  for(;;) {
1162  if(in_.eof())
1163  break;
1164  char d = static_cast<char>(in_.get());
1165  if(d == '\n')
1166  ++linenum_;
1167  buffer += d;
1168  if(d == '#') {
1169  if(in_.peek() == 'a') {
1170  found_arg = 1;
1171  } else if(in_.peek() == 'd') {
1172  found_deprecate = 1;
1173  } else {
1174  found_enddef = 1;
1175  }
1176  } else {
1177  if(found_arg > 0 && ++found_arg == 4) {
1178  if(std::equal(buffer.end() - 3, buffer.end(), "arg")) {
1179  buffer.erase(buffer.end() - 4, buffer.end());
1180 
1181  skip_spaces();
1182  std::string argname = read_word();
1183  skip_eol();
1184 
1185  std::string argbuffer;
1186 
1187  int found_endarg = 0;
1188  for(;;) {
1189  if(in_.eof()) {
1190  break;
1191  }
1192 
1193  char e = static_cast<char>(in_.get());
1194  if(e == '\n') {
1195  ++linenum_;
1196  }
1197 
1198  argbuffer += e;
1199 
1200  if(e == '#') {
1201  found_endarg = 1;
1202  } else if(found_endarg > 0 && ++found_endarg == 7) {
1203  if(std::equal(argbuffer.end() - 6, argbuffer.end(), "endarg")) {
1204  argbuffer.erase(argbuffer.end() - 7, argbuffer.end());
1205  optargs[argname] = argbuffer;
1206  skip_eol();
1207  break;
1208  } else {
1209  parent_.error("Unterminated #arg definition", linenum_);
1210  }
1211  }
1212  }
1213  }
1214  }
1215 
1216  if(found_deprecate > 0 && ++found_deprecate == 11) {
1217  if(std::equal(buffer.end() - 10, buffer.end(), "deprecated")) {
1218  buffer.erase(buffer.end() - 11, buffer.end());
1219  skip_spaces();
1220  try {
1221  DEP_LEVEL level = DEP_LEVEL(std::stoi(read_word()));
1222  if(deprecation_level) {
1223  deprecation_level = std::max(*deprecation_level, level);
1224  } else {
1225  deprecation_level = level;
1226  }
1227  } catch(const std::invalid_argument&) {
1228  // Meh, fall back to default of PREEMPTIVE...
1229  deprecation_level = DEP_LEVEL::PREEMPTIVE;
1230  }
1231  deprecation_version = game_config::wesnoth_version;
1232  if(deprecation_level == DEP_LEVEL::PREEMPTIVE || deprecation_level == DEP_LEVEL::FOR_REMOVAL) {
1233  skip_spaces();
1234  deprecation_version = std::max(deprecation_version, version_info(read_word()));
1235  }
1236  skip_spaces();
1237  if(!deprecation_detail.empty()){
1238  deprecation_detail += '\n';
1239  }
1240  deprecation_detail = read_rest_of_line();
1241  skip_eol();
1242  }
1243  }
1244 
1245  if(found_enddef > 0 && ++found_enddef == 7) {
1246  if(std::equal(buffer.end() - 6, buffer.end(), "enddef")) {
1247  break;
1248  } else {
1249  found_enddef = 0;
1250  if(std::equal(buffer.end() - 6, buffer.end(), "define")) { // TODO: Maybe add support for
1251  // this? This would fill feature
1252  // request #21343
1253  parent_.error(
1254  "Preprocessor error: #define is not allowed inside a #define/#enddef pair",
1255  linenum);
1256  }
1257  }
1258  }
1259  }
1260  }
1261 
1262  if(found_enddef != 7) {
1263  parent_.error("Unterminated preprocessor definition", linenum_);
1264  }
1265 
1266  if(!skipping_) {
1267  preproc_map::const_iterator old_i = parent_.defines_->find(symbol);
1268  if(old_i != parent_.defines_->end()) {
1269  std::ostringstream new_pos, old_pos;
1270  const preproc_define& old_d = old_i->second;
1271 
1272  new_pos << linenum << ' ' << parent_.location_;
1273  old_pos << old_d.linenum << ' ' << old_d.location;
1274 
1275  WRN_PREPROC << "Redefining macro " << symbol << " without explicit #undef at "
1276  << lineno_string(new_pos.str()) << '\n'
1277  << "previously defined at " << lineno_string(old_pos.str()) << '\n';
1278  }
1279 
1280  buffer.erase(buffer.end() - 7, buffer.end());
1281  (*parent_.defines_)[symbol]
1282  = preproc_define(buffer, items, optargs, parent_.textdomain_, linenum, parent_.location_,
1283  deprecation_detail, deprecation_level, deprecation_version);
1284 
1285  LOG_PREPROC << "defining macro " << symbol << " (location " << get_location(parent_.location_) << ")\n";
1286  }
1287  } else if(command == "ifdef") {
1288  skip_spaces();
1289  const std::string& symbol = read_word();
1290  bool found = parent_.defines_->count(symbol) != 0;
1291  DBG_PREPROC << "testing for macro " << symbol << ": " << (found ? "defined" : "not defined") << '\n';
1292  conditional_skip(!found);
1293  } else if(command == "ifndef") {
1294  skip_spaces();
1295  const std::string& symbol = read_word();
1296  bool found = parent_.defines_->count(symbol) != 0;
1297  DBG_PREPROC << "testing for macro " << symbol << ": " << (found ? "defined" : "not defined") << '\n';
1298  conditional_skip(found);
1299  } else if(command == "ifhave") {
1300  skip_spaces();
1301  const std::string& symbol = read_word();
1302  bool found = !filesystem::get_wml_location(symbol, directory_).empty();
1303  DBG_PREPROC << "testing for file or directory " << symbol << ": " << (found ? "found" : "not found")
1304  << '\n';
1305  conditional_skip(!found);
1306  } else if(command == "ifnhave") {
1307  skip_spaces();
1308  const std::string& symbol = read_word();
1309  bool found = !filesystem::get_wml_location(symbol, directory_).empty();
1310  DBG_PREPROC << "testing for file or directory " << symbol << ": " << (found ? "found" : "not found")
1311  << '\n';
1312  conditional_skip(found);
1313  } else if(command == "ifver" || command == "ifnver") {
1314  skip_spaces();
1315  const std::string& vsymstr = read_word();
1316  skip_spaces();
1317  const std::string& vopstr = read_word();
1318  skip_spaces();
1319  const std::string& vverstr = read_word();
1320 
1321  const VERSION_COMP_OP vop = parse_version_op(vopstr);
1322 
1323  if(vop == OP_INVALID) {
1324  parent_.error("Invalid #ifver/#ifnver operator", linenum_);
1325  } else if(parent_.defines_->count(vsymstr) != 0) {
1326  const preproc_define& sym = (*parent_.defines_)[vsymstr];
1327 
1328  if(!sym.arguments.empty()) {
1329  parent_.error("First argument macro in #ifver/#ifnver should not require arguments", linenum_);
1330  }
1331 
1332  version_info const version1(sym.value);
1333  version_info const version2(vverstr);
1334 
1335  const bool found = do_version_check(version1, vop, version2);
1336  DBG_PREPROC << "testing version '" << version1.str() << "' against '" << version2.str() << "' ("
1337  << vopstr << "): " << (found ? "match" : "no match") << '\n';
1338 
1339  conditional_skip(command == "ifver" ? !found : found);
1340  } else {
1341  std::string err = "Undefined macro in #ifver/#ifnver first argument: '";
1342  err += vsymstr;
1343  err += "'";
1344  parent_.error(err, linenum_);
1345  }
1346  } else if(command == "else") {
1347  if(token.type == token_desc::SKIP_ELSE) {
1348  pop_token();
1349  --skipping_;
1351  } else if(token.type == token_desc::PROCESS_IF) {
1352  pop_token();
1353  ++skipping_;
1355  } else {
1356  parent_.error("Unexpected #else", linenum_);
1357  }
1358  } else if(command == "endif") {
1359  switch(token.type) {
1360  case token_desc::SKIP_IF:
1361  case token_desc::SKIP_ELSE:
1362  --skipping_;
1365  break;
1366  default:
1367  parent_.error("Unexpected #endif", linenum_);
1368  }
1369  pop_token();
1370  } else if(command == "textdomain") {
1371  skip_spaces();
1372  const std::string& s = read_word();
1373  if(s != parent_.textdomain_) {
1374  put("#textdomain ");
1375  put(s);
1376  parent_.textdomain_ = s;
1377  }
1378  comment = true;
1379  } else if(command == "enddef") {
1380  parent_.error("Unexpected #enddef", linenum_);
1381  } else if(command == "undef") {
1382  skip_spaces();
1383  const std::string& symbol = read_word();
1384  if(!skipping_) {
1385  parent_.defines_->erase(symbol);
1386  LOG_PREPROC << "undefine macro " << symbol << " (location " << get_location(parent_.location_) << ")\n";
1387  }
1388  } else if(command == "error") {
1389  if(!skipping_) {
1390  skip_spaces();
1391  std::ostringstream error;
1392  error << "#error: \"" << read_rest_of_line() << '"';
1393  parent_.error(error.str(), linenum_);
1394  } else
1395  DBG_PREPROC << "Skipped an error\n";
1396  } else if(command == "warning") {
1397  if(!skipping_) {
1398  skip_spaces();
1399  std::ostringstream warning;
1400  warning << "#warning: \"" << read_rest_of_line() << '"';
1401  parent_.warning(warning.str(), linenum_);
1402  } else {
1403  DBG_PREPROC << "Skipped a warning\n";
1404  }
1405  } else if(command == "deprecated") {
1406  // The current file is deprecated, so print a message
1407  skip_spaces();
1409  try {
1410  level = DEP_LEVEL(std::stoi(read_word()));
1411  } catch(const std::invalid_argument&) {
1412  // Meh, just fall back to the default of PREEMPTIVE...
1413  }
1415  if(level == DEP_LEVEL::PREEMPTIVE || level == DEP_LEVEL::FOR_REMOVAL) {
1416  skip_spaces();
1417  version = version_info(read_word());
1418  }
1419  skip_spaces();
1420  std::string detail = read_rest_of_line();
1421  deprecated_message(get_filename(parent_.location_), level, version, detail);
1422  } else {
1423  comment = token.type != token_desc::MACRO_SPACE;
1424  }
1425 
1426  skip_eol();
1427  if(comment) {
1428  put('\n');
1429  }
1430  } else if(token.type == token_desc::MACRO_SPACE || token.type == token_desc::MACRO_CHUNK) {
1431  if(c == '(') {
1432  // If a macro argument was started, it is implicitly ended.
1433  token.type = token_desc::MACRO_SPACE;
1435  } else if(utils::portable_isspace(c)) {
1436  // If a macro argument was started, it is implicitly ended.
1437  token.type = token_desc::MACRO_SPACE;
1438  } else if(c == '}') {
1439  --slowpath_;
1440  if(skipping_) {
1441  pop_token();
1442  return true;
1443  }
1444 
1445  // FIXME: is this obsolete?
1446  // if (token.type == token_desc::MACRO_SPACE) {
1447  // if (!strings_.back().empty()) {
1448  // std::ostringstream error;
1449  // std::ostringstream location;
1450  // error << "Can't parse new macro parameter with a macro call scope open";
1451  // location<<linenum_<<' '<<parent_.location_;
1452  // parent_.error(error.str(), location.str());
1453  // }
1454  // strings_.pop_back();
1455  //}
1456 
1457  if(strings_.size() <= static_cast<std::size_t>(token.stack_pos)) {
1458  parent_.error("No macro or file substitution target specified", linenum_);
1459  }
1460 
1461  std::string symbol = strings_[token.stack_pos];
1462  std::string::size_type pos;
1463  while((pos = symbol.find(OUTPUT_SEPARATOR)) != std::string::npos) {
1464  std::string::iterator b = symbol.begin(); // invalidated at each iteration
1465  symbol.erase(b + pos, b + symbol.find('\n', pos + 1) + 1);
1466  }
1467 
1468  std::map<std::string, std::string>::const_iterator arg;
1469  preproc_map::const_iterator macro;
1470 
1471  // If this is a known pre-processing symbol, then we insert it,
1472  // otherwise we assume it's a file name to load.
1473  if(symbol == current_file_str && strings_.size() - token.stack_pos == 1) {
1474  pop_token();
1476  } else if(symbol == current_dir_str && strings_.size() - token.stack_pos == 1) {
1477  pop_token();
1479  } else if(local_defines_ && (arg = local_defines_->find(symbol)) != local_defines_->end()) {
1480  if(strings_.size() - token.stack_pos != 1) {
1481  std::ostringstream error;
1482  error << "Macro argument '" << symbol << "' does not expect any arguments";
1483  parent_.error(error.str(), linenum_);
1484  }
1485 
1486  std::ostringstream v;
1487  v << arg->second << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1488  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1489 
1490  pop_token();
1491  put(v.str());
1492  } else if(parent_.depth() < 100 && (macro = parent_.defines_->find(symbol)) != parent_.defines_->end()) {
1493  const preproc_define& val = macro->second;
1494  std::size_t nb_arg = strings_.size() - token.stack_pos - 1;
1495  std::size_t optional_arg_num = 0;
1496 
1497  std::unique_ptr<std::map<std::string, std::string>> defines{new std::map<std::string, std::string>};
1498  const std::string& dir = filesystem::directory_name(val.location.substr(0, val.location.find(' ')));
1499 
1500  if(val.is_deprecated()) {
1502  }
1503 
1504  for(std::size_t i = 0; i < nb_arg; ++i) {
1505  if(i < val.arguments.size()) {
1506  // Normal mandatory arguments
1507 
1508  (*defines)[val.arguments[i]] = strings_[token.stack_pos + i + 1];
1509  } else {
1510  // These should be optional argument overrides
1511 
1512  std::string str = strings_[token.stack_pos + i + 1];
1513  std::size_t equals_pos = str.find_first_of("=");
1514 
1515  if(equals_pos != std::string::npos) {
1516  std::size_t argname_pos = str.substr(0, equals_pos).find_last_of(" \n") + 1;
1517 
1518  std::string argname = str.substr(argname_pos, equals_pos - argname_pos);
1519 
1520  if(val.optional_arguments.find(argname) != val.optional_arguments.end()) {
1521  (*defines)[argname] = str.substr(equals_pos + 1);
1522 
1523  optional_arg_num++;
1524 
1525  DBG_PREPROC << "Found override for " << argname << " in call to macro " << symbol
1526  << "\n";
1527  } else {
1528  std::ostringstream warning;
1529  warning << "Unrecognized optional argument passed to macro '" << symbol << "': '"
1530  << argname << "'";
1531  parent_.warning(warning.str(), linenum_);
1532 
1533  optional_arg_num++; // To prevent the argument number check from blowing up
1534  }
1535  }
1536  }
1537  }
1538 
1539  // If the macro definition has any optional arguments, insert their defaults
1540  if(val.optional_arguments.size() > 0) {
1541  for(const auto& argument : val.optional_arguments) {
1542  if(defines->find(argument.first) == defines->end()) {
1543  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1544 
1546  std::istream in(buf.get());
1547 
1548  filesystem::scoped_istream buffer{new std::istringstream(argument.second)};
1549 
1550  std::unique_ptr<std::map<std::string, std::string>> temp_defines{new std::map<std::string, std::string>};
1551  temp_defines->insert(defines->begin(), defines->end());
1552 
1554  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(temp_defines), false);
1555 
1556  std::ostringstream res;
1557  res << in.rdbuf();
1558 
1559  DBG_PREPROC << "Setting default for optional argument " << argument.first << " in macro "
1560  << symbol << "\n";
1561 
1562  (*defines)[argument.first] = res.str();
1563  }
1564  }
1565  }
1566 
1567  if(nb_arg - optional_arg_num != val.arguments.size()) {
1568  const std::vector<std::string>& locations = utils::quoted_split(val.location, ' ');
1569  std::ostringstream error;
1570  error << "Preprocessor symbol '" << symbol << "' defined at " << get_filename(locations[0]) << ":"
1571  << val.linenum << " expects " << val.arguments.size() << " arguments, but has "
1572  << nb_arg - optional_arg_num << " arguments";
1573  parent_.error(error.str(), linenum_);
1574  }
1575 
1576  filesystem::scoped_istream buffer{new std::istringstream(val.value)};
1577 
1578  pop_token();
1579 
1580  if(!slowpath_) {
1581  DBG_PREPROC << "substituting macro " << symbol << '\n';
1582 
1584  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1585  } else {
1586  DBG_PREPROC << "substituting (slow) macro " << symbol << '\n';
1587 
1588  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1589 
1590  // Make the nested preprocessor_data responsible for
1591  // restoring our current textdomain if needed.
1593 
1594  std::ostringstream res;
1595  {
1596  std::istream in(buf.get());
1598  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1599 
1600  res << in.rdbuf();
1601  }
1602 
1603  put(res.str());
1604  }
1605  } else if(parent_.depth() < 40) {
1606  LOG_PREPROC << "Macro definition not found for " << symbol << " , attempting to open as file.\n";
1607  pop_token();
1608 
1609  std::string nfname = filesystem::get_wml_location(symbol, directory_);
1610  if(!nfname.empty()) {
1611  if(!slowpath_)
1612  // nfname.size() - symbol.size() gives you an index into nfname
1613  // This does not necessarily match the symbol though, as it can start with ~ or ./
1614  parent_.add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1615  else {
1616  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1617 
1618  std::ostringstream res;
1619  {
1620  std::istream in(buf.get());
1621  buf->add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1622 
1623  res << in.rdbuf();
1624  }
1625 
1626  put(res.str());
1627  }
1628  } else {
1629  std::ostringstream error;
1630  error << "Macro/file '" << symbol << "' is missing";
1631  parent_.error(error.str(), linenum_);
1632  }
1633  } else {
1634  parent_.error("Too many nested preprocessing inclusions", linenum_);
1635  }
1636  } else if(!skipping_) {
1637  if(token.type == token_desc::MACRO_SPACE) {
1638  std::ostringstream s;
1639  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1640  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1641 
1642  strings_.push_back(s.str());
1643  token.type = token_desc::MACRO_CHUNK;
1644  }
1645  put(c);
1646  }
1647  } else {
1648  put(c);
1649  }
1650 
1651  return true;
1652 }
1653 
1654 
1655 // ==================================================================================
1656 // PREPROCESSOR SCOPE HELPER
1657 // ==================================================================================
1658 
1659 struct preprocessor_scope_helper : std::basic_istream<char>
1660 {
1661  preprocessor_scope_helper(const std::string& fname, preproc_map* defines)
1662  : std::basic_istream<char>(nullptr)
1663  , buf_(nullptr)
1664  , local_defines_(nullptr)
1665  {
1666  //
1667  // If no defines were provided, we create a new local preproc_map and assign
1668  // it to defines temporarily. In this case, the map will be deleted once this
1669  // object is destroyed and defines will still be subsequently null.
1670  //
1671  if(!defines) {
1672  local_defines_.reset(new preproc_map);
1673  defines = local_defines_.get();
1674  }
1675 
1676  buf_.reset(new preprocessor_streambuf(defines));
1677 
1678  // Begin processing.
1679  buf_->add_preprocessor<preprocessor_file>(fname);
1680 
1681  //
1682  // TODO: not sure if this call is needed. Previously, this call was passed a
1683  // preprocessor_streambuf pointer and the std::basic_istream constructor was
1684  // called with its contents. However, at that point the preprocessing should
1685  // already have completed, meaning this call might be redundant. Not sure.
1686  //
1687  // - vultraz, 2017-08-31
1688  //
1689  init(buf_.get());
1690  }
1691 
1693  {
1694  clear(std::ios_base::goodbit);
1695  exceptions(std::ios_base::goodbit);
1696  rdbuf(nullptr);
1697  }
1698 
1699  std::unique_ptr<preprocessor_streambuf> buf_;
1700  std::unique_ptr<preproc_map> local_defines_;
1701 };
1702 
1703 
1704 // ==================================================================================
1705 // FREE-STANDING FUNCTIONS
1706 // ==================================================================================
1707 
1708 filesystem::scoped_istream preprocess_file(const std::string& fname, preproc_map* defines)
1709 {
1710  log_scope("preprocessing file " + fname + " ...");
1711 
1712  // NOTE: the preprocessor_scope_helper does *not* take ownership of defines.
1713  return filesystem::scoped_istream(new preprocessor_scope_helper(fname, defines));
1714 }
1715 
1716 void preprocess_resource(const std::string& res_name,
1717  preproc_map* defines_map,
1718  bool write_cfg,
1719  bool write_plain_cfg,
1720  const std::string& parent_directory)
1721 {
1722  if(filesystem::is_directory(res_name)) {
1723  std::vector<std::string> dirs, files;
1724 
1727 
1728  // Subdirectories
1729  for(const std::string& dir : dirs) {
1730  LOG_PREPROC << "processing sub-dir: " << dir << '\n';
1731  preprocess_resource(dir, defines_map, write_cfg, write_plain_cfg, parent_directory);
1732  }
1733 
1734  // Files in current directory
1735  for(const std::string& file : files) {
1736  preprocess_resource(file, defines_map, write_cfg, write_plain_cfg, parent_directory);
1737  }
1738 
1739  return;
1740  }
1741 
1742  // process only config files.
1743  if(!filesystem::ends_with(res_name, ".cfg")) {
1744  return;
1745  }
1746 
1747  LOG_PREPROC << "processing resource: " << res_name << '\n';
1748 
1749  // disable filename encoding to get clear #line in cfg.plain
1750  encode_filename = false;
1751 
1752  filesystem::scoped_istream stream = preprocess_file(res_name, defines_map);
1753 
1754  std::stringstream ss;
1755 
1756  // Set the failbit so if we get any preprocessor exceptions (e.g.:preproc_config::error)
1757  // they will be propagated in the main program, instead of just setting the
1758  // failbit on the stream. This was necessary in order for the MSVC and GCC
1759  // binaries to behave the same way.
1760  ss.exceptions(std::ios_base::failbit);
1761 
1762  ss << (*stream).rdbuf();
1763 
1764  LOG_PREPROC << "processing finished\n";
1765 
1766  if(write_cfg || write_plain_cfg) {
1767  config cfg;
1768  std::string streamContent = ss.str();
1769 
1770  read(cfg, streamContent);
1771 
1772  const std::string preproc_res_name = parent_directory + "/" + filesystem::base_name(res_name);
1773 
1774  // Write the processed cfg file
1775  if(write_cfg) {
1776  LOG_PREPROC << "writing cfg file: " << preproc_res_name << '\n';
1777 
1779  filesystem::scoped_ostream outStream(filesystem::ostream_file(preproc_res_name));
1780 
1781  write(*outStream, cfg);
1782  }
1783 
1784  // Write the plain cfg file
1785  if(write_plain_cfg) {
1786  LOG_PREPROC << "writing plain cfg file: " << (preproc_res_name + ".plain") << '\n';
1787 
1789  filesystem::write_file(preproc_res_name + ".plain", streamContent);
1790  }
1791  }
1792 }
static std::string current_file_str
std::string lineno_string(const std::string &lineno)
VERSION_COMP_OP
static const char OUTPUT_SEPARATOR
std::vector< std::string > arguments
preprocessor * current() const
preprocessor_file(preprocessor_streambuf &t, const std::string &name, std::size_t symbol_index=-1)
Constructor.
void write(config_writer &, const std::string &) const
std::vector< std::string > files_
virtual bool get_chunk() override
Inserts and processes the next file in the list of included files.
preprocessor_scope_helper(const std::string &fname, preproc_map *defines)
Specialized preprocessor for handling a file or a set of files.
void read(const config &)
Interfaces for manipulating version numbers of engine, add-ons, etc.
static int read_line(lua_State *L, FILE *f, int chop)
Definition: liolib.cpp:482
std::vector< std::string >::const_iterator end_
VERSION_COMP_OP parse_version_op(const std::string &op_str)
std::string read_word()
preproc_map default_defines_
static l_noret error(LoadState *S, const char *why)
Definition: lundump.cpp:39
buffered_istream in_
Input stream.
static std::map< std::string, int > file_number_map
bool ends_with(const std::string &str, const std::string &suffix)
#define ERR_PREPROC
child_itors child_range(config_key_type key)
Definition: config.cpp:366
virtual MODE parse_mode()
Returns the appropriate parsing mode for this preprocessor.
int peek()
Gets a character from the buffer.
Specialized preprocessor for handling any kind of input stream.
std::string out_buffer_
Buffer read by the STL stream.
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
Definition: filesystem.cpp:882
void clear(const std::string &key)
Definition: general.cpp:205
Helper class for buffering a std::istream.
virtual bool get_chunk() override
Preprocesses and sends some text to the parent_ buffer.
void preprocess_resource(const std::string &res_name, preproc_map *defines_map, bool write_cfg, bool write_plain_cfg, const std::string &parent_directory)
STL namespace.
void read_argument(const config &)
const std::vector< std::string > items
#define d
std::unique_ptr< preprocessor_streambuf > buf_
bool do_version_check(const version_info &a, VERSION_COMP_OP op, const version_info &b)
Definitions for the interface to Wesnoth Markup Language (WML).
std::vector< std::string > split(const std::string &val, const char c, const int flags)
Splits a (comma-)separated string into a vector of pieces.
std::vector< std::string >::const_iterator pos_
std::string location
Description of a preprocessing chunk.
#define WRN_PREPROC
#define b
void write_argument(config_writer &, const std::string &) const
unsigned in
If equal to search_counter, the node is off the list.
static const char * name(const std::vector< SDL_Joystick *> &joysticks, const std::size_t index)
Definition: joystick.cpp:48
void write(std::ostream &out, const configr_of &cfg, unsigned int level)
Definition: parser.cpp:752
int slowpath_
Set to true whenever input tokens cannot be directly sent to the target buffer.
bool is_define_
True iff we are currently parsing a macros content, otherwise false.
void write_key_val(const std::string &key, const T &value)
This template function will work with any type that can be assigned to an attribute_value.
int get()
Gets and consumes a character from the buffer.
void conditional_skip(bool skip)
std::string deprecated_message(const std::string &elem_name, DEP_LEVEL level, const version_info &version, const std::string &detail)
Definition: deprecation.cpp:29
bool quoted_
Set to true if one preprocessor for this target started to read a string.
void close_child(const std::string &key)
void add_preprocessor(A &&... args)
static lg::log_domain log_preprocessor("preprocessor")
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:615
Class for writing a config out to a file in pieces.
version_info deprecation_version
void restore_old_preprocessor()
Restores the old preprocessing context.
std::string read_line()
void write_file(const std::string &fname, const std::string &data)
Throws io_exception if an error occurs.
Definition: filesystem.cpp:943
int skipping_
Non-zero when the preprocessor has to skip some input text.
void push_token(token_desc::TOKEN_TYPE)
void open_child(const std::string &key)
std::deque< std::unique_ptr< preprocessor > > preprocessor_queue_
Input preprocessor queue.
virtual ~preprocessor()
bool is_deprecated() const
bool is_directory(const std::string &fname)
Returns true if the given file is a directory.
Definition: filesystem.cpp:972
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:39
virtual void init() override
Allows specifying any actions that need to be called after the constructor completes.
std::string get_short_wml_path(const std::string &filename)
Returns a short path to filename, skipping the (user) data directory.
std::string deprecation_message
#define DBG_PREPROC
std::string value
#define LOG_PREPROC
std::string escape(const std::string &str, const char *special_chars)
Prepends a configurable set of characters with a backslash.
#define PACKAGE
Definition: wesconfig.h:23
std::string directory_
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:40
virtual preprocessor::MODE parse_mode() override
Returns the appropriate parsing mode for this preprocessor.
static std::string get_file_code(const std::string &filename)
Some defines: VERSION, PACKAGE, MIN_SAVEGAME_VERSION.
virtual void init()
Allows specifying any actions that need to be called after the constructor completes.
bool operator==(const preproc_define &) const
token_desc(TOKEN_TYPE type, const int stack_pos, const int linenum)
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, file_name_option mode, file_filter_option filter, file_reorder_option reorder, file_tree_checksum *checksum)
Populates &#39;files&#39; with all the files and &#39;dirs&#39; with all the directories in dir.
Definition: filesystem.cpp:366
boost::optional< DEP_LEVEL > deprecation_level
std::size_t i
Definition: function.cpp:933
logger & err()
Definition: log.cpp:78
std::string get_wml_location(const std::string &filename, const std::string &current_dir)
Returns a complete path to the actual WML file or directory or an empty string if the file isn&#39;t pres...
std::string read_rest_of_line()
std::unique_ptr< preproc_map > local_defines_
bool operator!=(preprocessor_data::token_desc::TOKEN_TYPE rhs, char lhs)
mock_party p
static std::string get_location(const std::string &loc)
static map_location::DIRECTION s
filesystem::scoped_istream in_scope_
Manages the lifetime of the std::istream pointer we own.
std::vector< std::string > quoted_split(const std::string &val, char c, int flags, char quote)
This function is identical to split(), except it does not split when it otherwise would if the previo...
friend class preprocessor_streambuf
#define log_scope(description)
Definition: log.hpp:186
Helper class for buffering a std::istream.
std::unique_ptr< std::map< std::string, std::string > > local_defines_
Mapping of macro arguments to their content.
Declarations for File-IO.
static int writer(lua_State *L, const void *b, size_t size, void *B)
Definition: lstrlib.cpp:182
void warning(const std::string &, int)
const version_info wesnoth_version(VERSION)
Base class for preprocessing an input.
std::vector< std::string > strings_
Buffer for delayed input processing.
Represents version numbers.
std::ostream & operator<<(std::ostream &stream, const preproc_define &def)
static std::string current_dir_str
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
preprocessor_streambuf(const preprocessor_streambuf &t)
static std::string get_filename(const std::string &file_code)
double t
Definition: astarsearch.cpp:64
DEP_LEVEL
See https://wiki.wesnoth.org/CompatibilityStandards for more info.
Definition: deprecation.hpp:19
std::vector< token_desc > tokens_
Stack of nested preprocessing chunks.
preprocessor_streambuf & parent_
Standard logging facilities (interface).
std::string textdomain
std::string str() const
Serializes the version number into string form.
virtual int underflow() override
Inherited from basic_streambuf.
virtual bool get_chunk()=0
Preprocesses and sends some text to the parent_ buffer.
int stack_pos
Starting position in strings_ of the delayed text for this chunk.
preprocessor_data(preprocessor_streambuf &, filesystem::scoped_istream, const std::string &history, const std::string &name, int line, const std::string &dir, const std::string &domain, std::unique_ptr< std::map< std::string, std::string >> defines, bool is_define=false)
void error(const std::string &, int)
Target for sending preprocessed output.
std::map< std::string, struct preproc_define > preproc_map
#define e
bool portable_isspace(const char c)
preprocessor_streambuf(preproc_map *def)
bool operator==(preprocessor_data::token_desc::TOKEN_TYPE, char)
std::string get_current_file()
Decodes the filenames placed in a location.
std::string old_location_
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:92
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
static preproc_map::value_type read_pair(const config &)
mock_char c
const std::string & name_
std::string old_textdomain_
static map_location::DIRECTION n
bool operator<(const preproc_define &) const
static bool encode_filename
std::string::const_iterator iterator
Definition: tokenizer.hpp:24
std::stringstream buffer_
Buffer filled by the #current_ preprocessor.
preprocessor(preprocessor_streambuf &t)
Sets up a new preprocessor for stream buffer t.
std::string directory_name(const std::string &file)
Returns the directory name of a file, with filename stripped.
bool eof() const
Is the end of input reached?
static bool create_directory_if_missing_recursive(const bfs::path &dirpath)
Definition: filesystem.cpp:338
static std::string preprocessor_error_detail_prefix
filesystem::scoped_ostream ostream_file(const std::string &fname, bool create_directory)
Definition: filesystem.cpp:920
std::map< std::string, std::string > optional_arguments