The Battle for Wesnoth  1.15.0-dev
preprocessor.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2003 by David White <dave@whitevine.net>
3  Copyright (C) 2005 - 2018 by Guillaume Melquiond <guillaume.melquiond@gmail.com>
4  Part of the Battle for Wesnoth Project http://www.wesnoth.org/
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License as published by
8  the Free Software Foundation; either version 2 of the License, or
9  (at your option) any later version.
10  This program is distributed in the hope that it will be useful,
11  but WITHOUT ANY WARRANTY.
12 
13  See the COPYING file for more details.
14 */
15 
16 /**
17  * @file
18  * WML preprocessor.
19  */
20 
22 
23 #include "buffered_istream.hpp"
24 #include "config.hpp"
25 #include "deprecation.hpp"
26 #include "filesystem.hpp"
27 #include "log.hpp"
29 #include "serialization/parser.hpp"
31 #include "wesconfig.h"
32 
33 #include <deque>
34 #include <stdexcept>
35 
36 static lg::log_domain log_preprocessor("preprocessor");
37 #define ERR_PREPROC LOG_STREAM(err, log_preprocessor)
38 #define WRN_PREPROC LOG_STREAM(warn, log_preprocessor)
39 #define LOG_PREPROC LOG_STREAM(info, log_preprocessor)
40 #define DBG_PREPROC LOG_STREAM(debug, log_preprocessor)
41 
42 static std::string current_file_str = "CURRENT_FILE";
43 static std::string current_dir_str = "CURRENT_DIRECTORY";
44 
45 // map associating each filename encountered to a number
46 static std::map<std::string, int> file_number_map;
47 
48 static bool encode_filename = true;
49 
50 static std::string preprocessor_error_detail_prefix = "\n ";
51 
52 static const char OUTPUT_SEPARATOR = '\xFE';
53 
54 // get filename associated to this code
55 static std::string get_filename(const std::string& file_code)
56 {
57  if(!encode_filename) {
58  return file_code;
59  }
60 
61  std::stringstream s;
62  s << file_code;
63  int n = 0;
64  s >> std::hex >> n;
65 
66  for(const auto& p : file_number_map) {
67  if(p.second == n) {
68  return p.first;
69  }
70  }
71 
72  return "<unknown>";
73 }
74 
75 // Get code associated to this filename
76 static std::string get_file_code(const std::string& filename)
77 {
78  if(!encode_filename) {
79  return filename;
80  }
81 
82  // Current number of encountered filenames
83  static int current_file_number = 0;
84 
85  int& fnum = file_number_map[utils::escape(filename, " \\")];
86  if(fnum == 0) {
87  fnum = ++current_file_number;
88  }
89 
90  std::ostringstream shex;
91  shex << std::hex << fnum;
92 
93  return shex.str();
94 }
95 
96 // decode the filenames placed in a location
97 static std::string get_location(const std::string& loc)
98 {
99  std::string res;
100  std::vector<std::string> pos = utils::quoted_split(loc, ' ');
101 
102  if(pos.empty()) {
103  return res;
104  }
105 
106  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
107  while(true) {
108  res += get_filename(*(i++));
109 
110  if(i == end) {
111  break;
112  }
113 
114  res += ' ';
115  res += *(i++);
116 
117  if(i == end) {
118  break;
119  }
120 
121  res += ' ';
122  }
123 
124  return res;
125 }
126 
127 
128 // ==================================================================================
129 // PREPROC_DEFINE IMPLEMENTATION
130 // ==================================================================================
131 
133 {
134  return value == v.value && arguments == v.arguments;
135 }
136 
138 {
139  if(location < v.location) {
140  return true;
141  }
142 
143  if(v.location < location) {
144  return false;
145  }
146 
147  return linenum < v.linenum;
148 }
149 
150 void preproc_define::write_argument(config_writer& writer, const std::string& arg) const
151 {
152  const std::string key = "argument";
153 
154  writer.open_child(key);
155 
156  writer.write_key_val("name", arg);
157  writer.close_child(key);
158 }
159 
160 void preproc_define::write(config_writer& writer, const std::string& name) const
161 {
162  const std::string key = "preproc_define";
163  writer.open_child(key);
164 
165  writer.write_key_val("name", name);
166  writer.write_key_val("value", value);
167  writer.write_key_val("textdomain", textdomain);
168  writer.write_key_val("linenum", std::to_string(linenum));
169  writer.write_key_val("location", get_location(location));
170 
171  for(const std::string& arg : arguments) {
172  write_argument(writer, arg);
173  }
174 
175  writer.close_child(key);
176 }
177 
179 {
180  arguments.push_back(cfg["name"]);
181 }
182 
183 void preproc_define::read(const config& cfg)
184 {
185  value = cfg["value"].str();
186  textdomain = cfg["textdomain"].str();
187  linenum = cfg["linenum"];
188  location = cfg["location"].str();
189 
190  for(const config& arg : cfg.child_range("argument")) {
191  read_argument(arg);
192  }
193 }
194 
195 preproc_map::value_type preproc_define::read_pair(const config& cfg)
196 {
197  preproc_define second;
198  second.read(cfg);
199 
200  return preproc_map::value_type(cfg["name"], second);
201 }
202 
203 std::ostream& operator<<(std::ostream& stream, const preproc_define& def)
204 {
205  return stream << "value: " << def.value << " arguments: " << def.location;
206 }
207 
208 std::ostream& operator<<(std::ostream& stream, const preproc_map::value_type& def)
209 {
210  return stream << def.second;
211 }
212 
213 // ==================================================================================
214 // PREPROCESSOR BASE
215 // ==================================================================================
216 
218 
219 /**
220  * Base class for preprocessing an input.
221  */
223 {
225 
226 protected:
227  /**
228  * Sets up a new preprocessor for stream buffer \a t.
229  * Saves the current preprocessing context of #parent_. It will be automatically restored on destruction.
230  *
231  * It relies on preprocessor_streambuf so it's implemented after that class is declared.
232  */
234 
236 
237 public:
238  virtual ~preprocessor()
239  {
240  }
241 
242  /** Allows specifying any actions that need to be called after the constructor completes. */
243  virtual void init()
244  {
245  }
246 
247  /**
248  * Preprocesses and sends some text to the #parent_ buffer.
249  * @return false when the input has no data left.
250  */
251  virtual bool get_chunk() = 0;
252 
253  enum MODE { NO_PARSING, PARSES_FILE, PARSES_MACRO };
254 
255  /** Returns the appropriate parsing mode for this preprocessor. */
256  virtual MODE parse_mode()
257  {
258  return NO_PARSING;
259  }
260 
261 private:
262  std::string old_textdomain_;
263  std::string old_location_;
264 
266 };
267 
268 
269 // ==================================================================================
270 // PREPROCESSOR BUFFER
271 // ==================================================================================
272 
273 /**
274  * Target for sending preprocessed output.
275  * Objects of this class can be plugged into an STL stream.
276  */
277 class preprocessor_streambuf : public std::streambuf
278 {
279 public:
281  : std::streambuf()
282  , out_buffer_("")
283  , buffer_()
284  , preprocessor_queue_()
285  , defines_(def)
286  , default_defines_()
287  , textdomain_(PACKAGE)
288  , location_("")
289  , linenum_(0)
290  , quoted_(false)
291  {
292  }
293 
294  /** Decodes the filenames placed in a location. */
295  std::string get_current_file();
296 
297  void error(const std::string&, int);
298  void warning(const std::string&, int);
299 
300  template<typename T, typename... A>
301  void add_preprocessor(A&&... args)
302  {
303  preprocessor_queue_.emplace_back(new T(*this, std::forward<A>(args)...));
304  preprocessor_queue_.back()->init();
305  }
306 
308  {
309  preprocessor_queue_.pop_back();
310  }
311 
312  int depth() const
313  {
314  return preprocessor_queue_.size();
315  }
316 
318  {
319  return preprocessor_queue_.empty() ? nullptr : preprocessor_queue_.back().get();
320  }
321 
322 private:
324  : std::streambuf()
325  , out_buffer_("")
326  , buffer_()
327  , preprocessor_queue_()
328  , defines_(t.defines_)
329  , default_defines_()
330  , textdomain_(PACKAGE)
331  , location_("")
332  , linenum_(0)
333  , quoted_(t.quoted_)
334  {
335  }
336 
337  /** Inherited from basic_streambuf. */
338  virtual int underflow() override;
339 
340  void restore_old_preprocessor();
341 
342  /** Buffer read by the STL stream. */
343  std::string out_buffer_;
344 
345  /** Buffer filled by the #current_ preprocessor. */
346  std::stringstream buffer_;
347 
348  /** Input preprocessor queue. */
349  std::deque<std::unique_ptr<preprocessor>> preprocessor_queue_;
350 
353 
354  std::string textdomain_;
355  std::string location_;
356 
357  int linenum_;
358 
359  /**
360  * Set to true if one preprocessor for this target started to read a string.
361  * Deeper-nested preprocessors are then forbidden to.
362  */
363  bool quoted_;
364 
365  friend class preprocessor;
366  friend class preprocessor_file;
367  friend class preprocessor_data;
369 };
370 
371 /** Preprocessor constructor. */
373  : parent_(t)
374  , old_textdomain_(t.textdomain_)
375  , old_location_(t.location_)
376  , old_linenum_(t.linenum_)
377 {
378 }
379 
380 /**
381  * Called by an STL stream whenever it has reached the end of #out_buffer_.
382  * Fills #buffer_ by calling the #current_ preprocessor, then copies its
383  * content into #out_buffer_.
384  * @return the first character of #out_buffer_ if any, EOF otherwise.
385  */
387 {
388  unsigned sz = 0;
389  if(char* gp = gptr()) {
390  if(gp < egptr()) {
391  // Sanity check: the internal buffer has not been totally consumed,
392  // should we force the caller to use what remains first?
393  return *gp;
394  }
395 
396  // The buffer has been completely read; fill it again.
397  // Keep part of the previous buffer, to ensure putback capabilities.
398  sz = out_buffer_.size();
399  buffer_.str(std::string());
400 
401  if(sz > 3) {
402  buffer_ << out_buffer_.substr(sz - 3);
403  sz = 3;
404  } else {
405  buffer_ << out_buffer_;
406  }
407  } else {
408  // The internal get-data pointer is null
409  }
410 
411  const int desired_fill_amount = 2000;
412 
413  while(current() && buffer_.rdbuf()->in_avail() < desired_fill_amount) {
414  // Process files and data chunks until the desired buffer size is reached
415  if(!current()->get_chunk()) {
416  // Drop the current preprocessor item from the queue.
417  restore_old_preprocessor();
418  }
419  }
420 
421  // Update the internal state and data pointers
422  out_buffer_ = buffer_.str();
423  if(out_buffer_.empty()) {
424  return EOF;
425  }
426 
427  char* begin = &*out_buffer_.begin();
428  unsigned bs = out_buffer_.size();
429 
430  setg(begin, begin + sz, begin + bs);
431 
432  if(sz >= bs) {
433  return EOF;
434  }
435 
436  return static_cast<unsigned char>(*(begin + sz));
437 }
438 
439 /**
440 * Restores the old preprocessing context.
441 * Appends location and domain directives to the buffer, so that the parser
442 * notices these changes.
443 */
445 {
446  preprocessor* current = this->current();
447 
448  if(!current->old_location_.empty()) {
449  buffer_ << OUTPUT_SEPARATOR << "line " << current->old_linenum_ << ' ' << current->old_location_ << '\n';
450  }
451 
452  if(!current->old_textdomain_.empty() && textdomain_ != current->old_textdomain_) {
453  buffer_ << OUTPUT_SEPARATOR << "textdomain " << current->old_textdomain_ << '\n';
454  }
455 
456  location_ = current->old_location_;
457  linenum_ = current->old_linenum_;
458  textdomain_ = current->old_textdomain_;
459 
460  // Drop the preprocessor from the queue.
461  drop_preprocessor();
462 }
463 
465 {
466  unsigned nested_level = 0;
467 
468  preprocessor* pre = nullptr;
469 
470  // Iterate backwards over queue to get the last non-macro preprocessor.
471  for(auto p = preprocessor_queue_.rbegin(); p != preprocessor_queue_.rend(); ++p) {
472  pre = p->get();
473 
474  if(!pre || pre->parse_mode() == preprocessor::PARSES_FILE) {
475  break;
476  }
477 
478  if(pre->parse_mode() == preprocessor::PARSES_MACRO) {
479  ++nested_level;
480  }
481  }
482 
483  std::string res;
484  std::vector<std::string> pos = utils::quoted_split(location_, ' ');
485 
486  if(pos.size() <= 2 * nested_level) {
487  return res;
488  }
489 
490  return get_filename(pos[2 * nested_level]);
491 }
492 
493 std::string lineno_string(const std::string& lineno)
494 {
495  std::vector<std::string> pos = utils::quoted_split(lineno, ' ');
496  std::vector<std::string>::const_iterator i = pos.begin(), end = pos.end();
497  std::string included_from = preprocessor_error_detail_prefix + "included from ";
498  std::string res;
499 
500  while(i != end) {
501  const std::string& line = *(i++);
502 
503  if(!res.empty()) {
504  res += included_from;
505  }
506 
507  if(i != end) {
508  res += get_filename(*(i++));
509  } else {
510  res += "<unknown>";
511  }
512 
513  res += ':' + line;
514  }
515 
516  if(res.empty()) {
517  res = "???";
518  }
519 
520  return res;
521 }
522 
523 void preprocessor_streambuf::error(const std::string& error_type, int l)
524 {
525  std::string position, error;
526  std::ostringstream pos;
527 
528  pos << l << ' ' << location_;
529  position = lineno_string(pos.str());
530 
531  error = error_type + '\n';
532  error += "at " + position;
533 
534  ERR_PREPROC << error << '\n';
535 
536  throw preproc_config::error(error);
537 }
538 
539 void preprocessor_streambuf::warning(const std::string& warning_type, int l)
540 {
541  std::string position, warning;
542  std::ostringstream pos;
543 
544  pos << l << ' ' << location_;
545  position = lineno_string(pos.str());
546 
547  warning = warning_type + '\n';
548  warning += "at " + position;
549 
550  WRN_PREPROC << warning << '\n';
551 }
552 
553 
554 // ==================================================================================
555 // PREPROCESSOR FILE
556 // ==================================================================================
557 
558 /**
559  * Specialized preprocessor for handling a file or a set of files.
560  * A preprocessor_file object is created when a preprocessor encounters an
561  * inclusion directive that resolves to a file or directory, e.g. '{themes/}'.
562  */
564 {
565 public:
566  /** Constructor. It relies on preprocessor_data so it's implemented after that class is declared. */
567  preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index = -1);
568 
569  virtual void init() override;
570 
571  /**
572  * Inserts and processes the next file in the list of included files.
573  * @return false if there is no next file.
574  */
575  virtual bool get_chunk() override
576  {
577  while(pos_ != end_) {
578  const std::string& name = *(pos_++);
579  unsigned sz = name.size();
580 
581  // Use reverse iterator to optimize testing
582  if(sz < 5 || !std::equal(name.rbegin(), name.rbegin() + 4, "gfc.")) {
583  continue;
584  }
585 
587  return true;
588  }
589 
590  return false;
591  }
592 
593 private:
594  std::vector<std::string> files_;
595  std::vector<std::string>::const_iterator pos_, end_;
596 
597  const std::string& name_;
598 
600 };
601 
602 
603 // ==================================================================================
604 // PREPROCESSOR DATA
605 // ==================================================================================
606 
607 /**
608  * Specialized preprocessor for handling any kind of input stream.
609  * This is the core of the preprocessor.
610  */
612 {
613  /** Description of a preprocessing chunk. */
614  struct token_desc
615  {
616  enum TOKEN_TYPE {
617  START, // Toplevel
618  PROCESS_IF, // Processing the "if" branch of a ifdef/ifndef (the "else" branch will be skipped)
619  PROCESS_ELSE, // Processing the "else" branch of a ifdef/ifndef
620  SKIP_IF, // Skipping the "if" branch of a ifdef/ifndef (the "else" branch, if any, will be processed)
621  SKIP_ELSE, // Skipping the "else" branch of a ifdef/ifndef
622  STRING, // Processing a string
623  VERBATIM, // Processing a verbatim string
624  MACRO_SPACE, // Processing between chunks of a macro call (skip spaces)
625  MACRO_CHUNK, // Processing inside a chunk of a macro call (stop on space or '(')
626  MACRO_PARENS // Processing a parenthesized macro argument
627  };
628 
629  token_desc(TOKEN_TYPE type, const int stack_pos, const int linenum)
630  : type(type)
631  , stack_pos(stack_pos)
632  , linenum(linenum)
633  {
634  }
635 
637 
638  /** Starting position in #strings_ of the delayed text for this chunk. */
640  int linenum;
641  };
642 
643  /**
644  * Manages the lifetime of the @c std::istream pointer we own.
645  *
646  * Since @ref in_ uses the stream as well this object must be created
647  * before @ref in_ and destroyed after @ref in_ is destroyed.
648  */
650 
651  /** Input stream. */
653 
654  std::string directory_;
655 
656  /** Buffer for delayed input processing. */
657  std::vector<std::string> strings_;
658 
659  /** Mapping of macro arguments to their content. */
660  std::unique_ptr<std::map<std::string, std::string>> local_defines_;
661 
662  /** Stack of nested preprocessing chunks. */
663  std::vector<token_desc> tokens_;
664 
665  /**
666  * Set to true whenever input tokens cannot be directly sent to the target
667  * buffer. For instance, this happens with macro arguments. In that case,
668  * the output is redirected toward #strings_ until it can be processed.
669  */
671 
672  /**
673  * Non-zero when the preprocessor has to skip some input text.
674  * Increased whenever entering a conditional branch that is not useful,
675  * e.g. a ifdef that evaluates to false.
676  */
678  int linenum_;
679 
680  /** True iff we are currently parsing a macros content, otherwise false. */
682 
683  std::string read_word();
684  std::string read_line();
685  std::string read_rest_of_line();
686 
687  void skip_spaces();
688  void skip_eol();
689  void push_token(token_desc::TOKEN_TYPE);
690  void pop_token();
691  void put(char);
692  void put(const std::string& /*, int change_line = 0 */);
693  void conditional_skip(bool skip);
694 
695 public:
698  const std::string& history,
699  const std::string& name,
700  int line,
701  const std::string& dir,
702  const std::string& domain,
703  std::unique_ptr<std::map<std::string, std::string>> defines,
704  bool is_define = false);
705 
706  virtual bool get_chunk() override;
707 
708  virtual preprocessor::MODE parse_mode() override
709  {
710  return is_define_ ? PARSES_MACRO : PARSES_FILE;
711  }
712 
717 };
718 
720 {
721  throw std::logic_error("don't compare tokens with characters");
722 }
723 
725 {
726  return rhs == lhs;
727 }
728 
730 {
731  return !(lhs == rhs);
732 }
733 
735 {
736  return rhs != lhs;
737 }
738 
739 /** preprocessor_file constructor. */
740 preprocessor_file::preprocessor_file(preprocessor_streambuf& t, const std::string& name, std::size_t symbol_index)
741  : preprocessor(t)
742  , files_()
743  , pos_()
744  , end_()
745  , name_(name)
746  , is_directory_(filesystem::is_directory(name))
747 {
748  if(is_directory_) {
749  filesystem::get_files_in_dir(name, &files_, nullptr,
753  );
754 
755  for(const std::string& fname : files_) {
756  std::size_t cpos = fname.rfind(" ");
757 
758  if(cpos != std::string::npos && cpos >= symbol_index) {
759  std::stringstream ss;
760  ss << "Found filename containing whitespace: '" << filesystem::base_name(fname)
761  << "' in included directory '" << name << "'.\nThe included symbol probably looks similar to '"
762  << filesystem::directory_name(fname.substr(symbol_index)) << "'";
763 
764  // TODO: find a real linenumber
765  parent_.error(ss.str(), -1);
766  }
767  }
768  } else {
769  // Handled in the init() function.
770  }
771 
772  pos_ = files_.begin();
773  end_ = files_.end();
774 }
775 
777 {
778  if(is_directory_) {
779  return;
780  }
781 
783 
784  if(!file_stream->good()) {
785  ERR_PREPROC << "Could not open file " << name_ << std::endl;
786  } else {
787  parent_.add_preprocessor<preprocessor_data>(std::move(file_stream), "",
790  );
791  }
792 }
793 
796  const std::string& history,
797  const std::string& name,
798  int linenum,
799  const std::string& directory,
800  const std::string& domain,
801  std::unique_ptr<std::map<std::string, std::string>> defines,
802  bool is_define)
803  : preprocessor(t)
804  , in_scope_(std::move(i))
805  , in_(*in_scope_)
806  , directory_(directory)
807  , strings_()
808  , local_defines_(std::move(defines))
809  , tokens_()
810  , slowpath_(0)
811  , skipping_(0)
812  , linenum_(linenum)
813  , is_define_(is_define)
814 {
815  std::ostringstream s;
816  s << history;
817 
818  if(!name.empty()) {
819  if(!history.empty()) {
820  s << ' ';
821  }
822 
823  s << get_file_code(name);
824  }
825 
826  if(!t.location_.empty()) {
827  s << ' ' << t.linenum_ << ' ' << t.location_;
828  }
829 
830  t.location_ = s.str();
831  t.linenum_ = linenum;
832 
833  t.buffer_ << OUTPUT_SEPARATOR << "line " << linenum << ' ' << t.location_ << '\n';
834 
835  if(t.textdomain_ != domain) {
836  t.buffer_ << OUTPUT_SEPARATOR << "textdomain " << domain << '\n';
837  t.textdomain_ = domain;
838  }
839 
841 }
842 
844 {
845  tokens_.emplace_back(t, strings_.size(), linenum_);
846 
847  if(t == token_desc::MACRO_SPACE) {
848  // Macro expansions do not have any associated storage at start.
849  return;
850  } else if(t == token_desc::STRING || t == token_desc::VERBATIM) {
851  /* Quoted strings are always inlined in the parent token. So
852  * they need neither storage nor metadata, unless the parent
853  * token is a macro expansion.
854  */
855  token_desc::TOKEN_TYPE& outer_type = tokens_[tokens_.size() - 2].type;
856  if(outer_type != token_desc::MACRO_SPACE) {
857  return;
858  }
859 
860  outer_type = token_desc::MACRO_CHUNK;
861  tokens_.back().stack_pos = strings_.size() + 1;
862  }
863 
864  std::ostringstream s;
865  if(!skipping_ && slowpath_) {
866  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
867  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
868  }
869 
870  strings_.push_back(s.str());
871 }
872 
874 {
875  token_desc::TOKEN_TYPE inner_type = tokens_.back().type;
876  unsigned stack_pos = tokens_.back().stack_pos;
877 
878  tokens_.pop_back();
879 
880  token_desc::TOKEN_TYPE& outer_type = tokens_.back().type;
881 
882  if(inner_type == token_desc::MACRO_PARENS) {
883  // Parenthesized macro arguments are left on the stack.
884  assert(outer_type == token_desc::MACRO_SPACE);
885  return;
886  }
887 
888  if(inner_type == token_desc::STRING || inner_type == token_desc::VERBATIM) {
889  // Quoted strings are always inlined.
890  assert(stack_pos == strings_.size());
891  return;
892  }
893 
894  if(outer_type == token_desc::MACRO_SPACE) {
895  /* A macro expansion does not have any associated storage.
896  * Instead, storage of the inner token is not discarded
897  * but kept as a new macro argument. But if the inner token
898  * was a macro expansion, it is about to be appended, so
899  * prepare for it.
900  */
901  if(inner_type == token_desc::MACRO_SPACE || inner_type == token_desc::MACRO_CHUNK) {
902  strings_.erase(strings_.begin() + stack_pos, strings_.end());
903  strings_.emplace_back();
904  }
905 
906  assert(stack_pos + 1 == strings_.size());
907  outer_type = token_desc::MACRO_CHUNK;
908 
909  return;
910  }
911 
912  strings_.erase(strings_.begin() + stack_pos, strings_.end());
913 }
914 
916 {
917  for(;;) {
918  int c = in_.peek();
919 
920  if(in_.eof() || (c != ' ' && c != '\t')) {
921  return;
922  }
923 
924  in_.get();
925  }
926 }
927 
929 {
930  for(;;) {
931  int c = in_.get();
932 
933  if(c == '\n') {
934  ++linenum_;
935  return;
936  }
937 
938  if(in_.eof()) {
939  return;
940  }
941  }
942 }
943 
945 {
946  std::string res;
947 
948  for(;;) {
949  int c = in_.peek();
950 
951  if(c == preprocessor_streambuf::traits_type::eof() || utils::portable_isspace(c)) {
952  // DBG_PREPROC << "(" << res << ")\n";
953  return res;
954  }
955 
956  in_.get();
957  res += static_cast<char>(c);
958  }
959 }
960 
962 {
963  std::string res;
964 
965  for(;;) {
966  int c = in_.get();
967 
968  if(c == '\n') {
969  ++linenum_;
970  return res;
971  }
972 
973  if(in_.eof()) {
974  return res;
975  }
976 
977  if(c != '\r') {
978  res += static_cast<char>(c);
979  }
980  }
981 }
982 
984 {
985  std::string res;
986 
987  while(in_.peek() != '\n' && !in_.eof()) {
988  int c = in_.get();
989 
990  if(c != '\r') {
991  res += static_cast<char>(c);
992  }
993  }
994 
995  return res;
996 }
997 
999 {
1000  if(skipping_) {
1001  return;
1002  }
1003 
1004  if(slowpath_) {
1005  strings_.back() += c;
1006  return;
1007  }
1008 
1009  int cond_linenum = c == '\n' ? linenum_ - 1 : linenum_;
1010 
1011  if(unsigned diff = cond_linenum - parent_.linenum_) {
1012  parent_.linenum_ = cond_linenum;
1013 
1014  if(diff <= parent_.location_.size() + 11) {
1015  parent_.buffer_ << std::string(diff, '\n');
1016  } else {
1017  parent_.buffer_ << OUTPUT_SEPARATOR << "line " << parent_.linenum_ << ' ' << parent_.location_ << '\n';
1018  }
1019  }
1020 
1021  if(c == '\n') {
1022  ++parent_.linenum_;
1023  }
1024 
1025  parent_.buffer_ << c;
1026 }
1027 
1028 void preprocessor_data::put(const std::string& s /*, int line_change*/)
1029 {
1030  if(skipping_) {
1031  return;
1032  }
1033 
1034  if(slowpath_) {
1035  strings_.back() += s;
1036  return;
1037  }
1038 
1039  parent_.buffer_ << s;
1040  // parent_.linenum_ += line_change;
1041 }
1042 
1044 {
1045  if(skip) {
1046  ++skipping_;
1047  }
1048 
1050 }
1051 
1053 {
1054  char c = static_cast<char>(in_.get());
1055  token_desc& token = tokens_.back();
1056 
1057  if(in_.eof()) {
1058  // The end of file was reached.
1059  // Make sure we don't have any incomplete tokens.
1060  char const* s;
1061 
1062  switch(token.type) {
1063  case token_desc::START:
1064  return false; // everything is fine
1066  case token_desc::SKIP_IF:
1068  case token_desc::SKIP_ELSE:
1069  s = "#ifdef or #ifndef";
1070  break;
1071  case token_desc::STRING:
1072  s = "Quoted string";
1073  break;
1074  case token_desc::VERBATIM:
1075  s = "Verbatim string";
1076  break;
1079  s = "Macro substitution";
1080  break;
1082  s = "Macro argument";
1083  break;
1084  default:
1085  s = "???";
1086  }
1087 
1088  parent_.error(std::string(s) + " not terminated", token.linenum);
1089  }
1090 
1091  if(c == '\n') {
1092  ++linenum_;
1093  }
1094 
1095  if(c == OUTPUT_SEPARATOR) {
1096  std::string buffer(1, c);
1097 
1098  for(;;) {
1099  char d = static_cast<char>(in_.get());
1100 
1101  if(in_.eof() || d == '\n') {
1102  break;
1103  }
1104 
1105  buffer += d;
1106  }
1107 
1108  buffer += '\n';
1109  // line_change = 1-1 = 0
1110  put(buffer);
1111  } else if(token.type == token_desc::VERBATIM) {
1112  put(c);
1113 
1114  if(c == '>' && in_.peek() == '>') {
1115  put(in_.get());
1116  pop_token();
1117  }
1118  } else if(c == '<' && in_.peek() == '<') {
1119  in_.get();
1121  put('<');
1122  put('<');
1123  } else if(c == '"') {
1124  if(token.type == token_desc::STRING) {
1125  parent_.quoted_ = false;
1126  put(c);
1127  pop_token();
1128  } else if(!parent_.quoted_) {
1129  parent_.quoted_ = true;
1131  put(c);
1132  } else {
1133  parent_.error("Nested quoted string", linenum_);
1134  }
1135  } else if(c == '{') {
1137  ++slowpath_;
1138  } else if(c == ')' && token.type == token_desc::MACRO_PARENS) {
1139  pop_token();
1140  } else if(c == '#' && !parent_.quoted_) {
1141  std::string command = read_word();
1142  bool comment = false;
1143 
1144  if(command == "define") {
1145  skip_spaces();
1146  int linenum = linenum_;
1147  std::vector<std::string> items = utils::split(read_line(), ' ');
1148  std::map<std::string, std::string> optargs;
1149 
1150  if(items.empty()) {
1151  parent_.error("No macro name found after #define directive", linenum);
1152  }
1153 
1154  std::string symbol = items.front();
1155  items.erase(items.begin());
1156  int found_arg = 0, found_enddef = 0, found_deprecate = 0;
1157  boost::optional<DEP_LEVEL> deprecation_level = boost::none;
1158  std::string buffer, deprecation_detail;
1159  version_info deprecation_version = game_config::wesnoth_version;
1160  for(;;) {
1161  if(in_.eof())
1162  break;
1163  char d = static_cast<char>(in_.get());
1164  if(d == '\n')
1165  ++linenum_;
1166  buffer += d;
1167  if(d == '#') {
1168  if(in_.peek() == 'a') {
1169  found_arg = 1;
1170  } else if(in_.peek() == 'd') {
1171  found_deprecate = 1;
1172  } else {
1173  found_enddef = 1;
1174  }
1175  } else {
1176  if(found_arg > 0 && ++found_arg == 4) {
1177  if(std::equal(buffer.end() - 3, buffer.end(), "arg")) {
1178  buffer.erase(buffer.end() - 4, buffer.end());
1179 
1180  skip_spaces();
1181  std::string argname = read_word();
1182  skip_eol();
1183 
1184  std::string argbuffer;
1185 
1186  int found_endarg = 0;
1187  for(;;) {
1188  if(in_.eof()) {
1189  break;
1190  }
1191 
1192  char e = static_cast<char>(in_.get());
1193  if(e == '\n') {
1194  ++linenum_;
1195  }
1196 
1197  argbuffer += e;
1198 
1199  if(e == '#') {
1200  found_endarg = 1;
1201  } else if(found_endarg > 0 && ++found_endarg == 7) {
1202  if(std::equal(argbuffer.end() - 6, argbuffer.end(), "endarg")) {
1203  argbuffer.erase(argbuffer.end() - 7, argbuffer.end());
1204  optargs[argname] = argbuffer;
1205  skip_eol();
1206  break;
1207  } else {
1208  parent_.error("Unterminated #arg definition", linenum_);
1209  }
1210  }
1211  }
1212  }
1213  }
1214 
1215  if(found_deprecate > 0 && ++found_deprecate == 11) {
1216  if(std::equal(buffer.end() - 10, buffer.end(), "deprecated")) {
1217  buffer.erase(buffer.end() - 11, buffer.end());
1218  skip_spaces();
1219  try {
1220  DEP_LEVEL level = DEP_LEVEL(std::stoi(read_word()));
1221  if(deprecation_level) {
1222  deprecation_level = std::max(*deprecation_level, level);
1223  } else {
1224  deprecation_level = level;
1225  }
1226  } catch(const std::invalid_argument&) {
1227  // Meh, fall back to default of PREEMPTIVE...
1228  deprecation_level = DEP_LEVEL::PREEMPTIVE;
1229  }
1230  deprecation_version = game_config::wesnoth_version;
1231  if(deprecation_level == DEP_LEVEL::PREEMPTIVE || deprecation_level == DEP_LEVEL::FOR_REMOVAL) {
1232  skip_spaces();
1233  deprecation_version = std::max(deprecation_version, version_info(read_word()));
1234  }
1235  skip_spaces();
1236  if(!deprecation_detail.empty()){
1237  deprecation_detail += '\n';
1238  }
1239  deprecation_detail = read_rest_of_line();
1240  skip_eol();
1241  }
1242  }
1243 
1244  if(found_enddef > 0 && ++found_enddef == 7) {
1245  if(std::equal(buffer.end() - 6, buffer.end(), "enddef")) {
1246  break;
1247  } else {
1248  found_enddef = 0;
1249  if(std::equal(buffer.end() - 6, buffer.end(), "define")) { // TODO: Maybe add support for
1250  // this? This would fill feature
1251  // request #21343
1252  parent_.error(
1253  "Preprocessor error: #define is not allowed inside a #define/#enddef pair",
1254  linenum);
1255  }
1256  }
1257  }
1258  }
1259  }
1260 
1261  if(found_enddef != 7) {
1262  parent_.error("Unterminated preprocessor definition", linenum_);
1263  }
1264 
1265  if(!skipping_) {
1266  preproc_map::const_iterator old_i = parent_.defines_->find(symbol);
1267  if(old_i != parent_.defines_->end()) {
1268  std::ostringstream new_pos, old_pos;
1269  const preproc_define& old_d = old_i->second;
1270 
1271  new_pos << linenum << ' ' << parent_.location_;
1272  old_pos << old_d.linenum << ' ' << old_d.location;
1273 
1274  WRN_PREPROC << "Redefining macro " << symbol << " without explicit #undef at "
1275  << lineno_string(new_pos.str()) << '\n'
1276  << "previously defined at " << lineno_string(old_pos.str()) << '\n';
1277  }
1278 
1279  buffer.erase(buffer.end() - 7, buffer.end());
1280  (*parent_.defines_)[symbol]
1281  = preproc_define(buffer, items, optargs, parent_.textdomain_, linenum, parent_.location_,
1282  deprecation_detail, deprecation_level, deprecation_version);
1283 
1284  LOG_PREPROC << "defining macro " << symbol << " (location " << get_location(parent_.location_) << ")\n";
1285  }
1286  } else if(command == "ifdef") {
1287  skip_spaces();
1288  const std::string& symbol = read_word();
1289  bool found = parent_.defines_->count(symbol) != 0;
1290  DBG_PREPROC << "testing for macro " << symbol << ": " << (found ? "defined" : "not defined") << '\n';
1291  conditional_skip(!found);
1292  } else if(command == "ifndef") {
1293  skip_spaces();
1294  const std::string& symbol = read_word();
1295  bool found = parent_.defines_->count(symbol) != 0;
1296  DBG_PREPROC << "testing for macro " << symbol << ": " << (found ? "defined" : "not defined") << '\n';
1297  conditional_skip(found);
1298  } else if(command == "ifhave") {
1299  skip_spaces();
1300  const std::string& symbol = read_word();
1301  bool found = !filesystem::get_wml_location(symbol, directory_).empty();
1302  DBG_PREPROC << "testing for file or directory " << symbol << ": " << (found ? "found" : "not found")
1303  << '\n';
1304  conditional_skip(!found);
1305  } else if(command == "ifnhave") {
1306  skip_spaces();
1307  const std::string& symbol = read_word();
1308  bool found = !filesystem::get_wml_location(symbol, directory_).empty();
1309  DBG_PREPROC << "testing for file or directory " << symbol << ": " << (found ? "found" : "not found")
1310  << '\n';
1311  conditional_skip(found);
1312  } else if(command == "ifver" || command == "ifnver") {
1313  skip_spaces();
1314  const std::string& vsymstr = read_word();
1315  skip_spaces();
1316  const std::string& vopstr = read_word();
1317  skip_spaces();
1318  const std::string& vverstr = read_word();
1319 
1320  const VERSION_COMP_OP vop = parse_version_op(vopstr);
1321 
1322  if(vop == OP_INVALID) {
1323  parent_.error("Invalid #ifver/#ifnver operator", linenum_);
1324  } else if(parent_.defines_->count(vsymstr) != 0) {
1325  const preproc_define& sym = (*parent_.defines_)[vsymstr];
1326 
1327  if(!sym.arguments.empty()) {
1328  parent_.error("First argument macro in #ifver/#ifnver should not require arguments", linenum_);
1329  }
1330 
1331  version_info const version1(sym.value);
1332  version_info const version2(vverstr);
1333 
1334  const bool found = do_version_check(version1, vop, version2);
1335  DBG_PREPROC << "testing version '" << version1.str() << "' against '" << version2.str() << "' ("
1336  << vopstr << "): " << (found ? "match" : "no match") << '\n';
1337 
1338  conditional_skip(command == "ifver" ? !found : found);
1339  } else {
1340  std::string err = "Undefined macro in #ifver/#ifnver first argument: '";
1341  err += vsymstr;
1342  err += "'";
1343  parent_.error(err, linenum_);
1344  }
1345  } else if(command == "else") {
1346  if(token.type == token_desc::SKIP_ELSE) {
1347  pop_token();
1348  --skipping_;
1350  } else if(token.type == token_desc::PROCESS_IF) {
1351  pop_token();
1352  ++skipping_;
1354  } else {
1355  parent_.error("Unexpected #else", linenum_);
1356  }
1357  } else if(command == "endif") {
1358  switch(token.type) {
1359  case token_desc::SKIP_IF:
1360  case token_desc::SKIP_ELSE:
1361  --skipping_;
1364  break;
1365  default:
1366  parent_.error("Unexpected #endif", linenum_);
1367  }
1368  pop_token();
1369  } else if(command == "textdomain") {
1370  skip_spaces();
1371  const std::string& s = read_word();
1372  if(s != parent_.textdomain_) {
1373  put("#textdomain ");
1374  put(s);
1375  parent_.textdomain_ = s;
1376  }
1377  comment = true;
1378  } else if(command == "enddef") {
1379  parent_.error("Unexpected #enddef", linenum_);
1380  } else if(command == "undef") {
1381  skip_spaces();
1382  const std::string& symbol = read_word();
1383  if(!skipping_) {
1384  parent_.defines_->erase(symbol);
1385  LOG_PREPROC << "undefine macro " << symbol << " (location " << get_location(parent_.location_) << ")\n";
1386  }
1387  } else if(command == "error") {
1388  if(!skipping_) {
1389  skip_spaces();
1390  std::ostringstream error;
1391  error << "#error: \"" << read_rest_of_line() << '"';
1392  parent_.error(error.str(), linenum_);
1393  } else
1394  DBG_PREPROC << "Skipped an error\n";
1395  } else if(command == "warning") {
1396  if(!skipping_) {
1397  skip_spaces();
1398  std::ostringstream warning;
1399  warning << "#warning: \"" << read_rest_of_line() << '"';
1400  parent_.warning(warning.str(), linenum_);
1401  } else {
1402  DBG_PREPROC << "Skipped a warning\n";
1403  }
1404  } else if(command == "deprecated") {
1405  // The current file is deprecated, so print a message
1406  skip_spaces();
1408  try {
1409  level = DEP_LEVEL(std::stoi(read_word()));
1410  } catch(const std::invalid_argument&) {
1411  // Meh, just fall back to the default of PREEMPTIVE...
1412  }
1414  if(level == DEP_LEVEL::PREEMPTIVE || level == DEP_LEVEL::FOR_REMOVAL) {
1415  skip_spaces();
1416  version = version_info(read_word());
1417  }
1418  skip_spaces();
1419  std::string detail = read_rest_of_line();
1420  deprecated_message(get_filename(parent_.location_), level, version, detail);
1421  } else {
1422  comment = token.type != token_desc::MACRO_SPACE;
1423  }
1424 
1425  skip_eol();
1426  if(comment) {
1427  put('\n');
1428  }
1429  } else if(token.type == token_desc::MACRO_SPACE || token.type == token_desc::MACRO_CHUNK) {
1430  if(c == '(') {
1431  // If a macro argument was started, it is implicitly ended.
1432  token.type = token_desc::MACRO_SPACE;
1434  } else if(utils::portable_isspace(c)) {
1435  // If a macro argument was started, it is implicitly ended.
1436  token.type = token_desc::MACRO_SPACE;
1437  } else if(c == '}') {
1438  --slowpath_;
1439  if(skipping_) {
1440  pop_token();
1441  return true;
1442  }
1443 
1444  // FIXME: is this obsolete?
1445  // if (token.type == token_desc::MACRO_SPACE) {
1446  // if (!strings_.back().empty()) {
1447  // std::ostringstream error;
1448  // std::ostringstream location;
1449  // error << "Can't parse new macro parameter with a macro call scope open";
1450  // location<<linenum_<<' '<<parent_.location_;
1451  // parent_.error(error.str(), location.str());
1452  // }
1453  // strings_.pop_back();
1454  //}
1455 
1456  if(strings_.size() <= static_cast<std::size_t>(token.stack_pos)) {
1457  parent_.error("No macro or file substitution target specified", linenum_);
1458  }
1459 
1460  std::string symbol = strings_[token.stack_pos];
1461  std::string::size_type pos;
1462  while((pos = symbol.find(OUTPUT_SEPARATOR)) != std::string::npos) {
1463  std::string::iterator b = symbol.begin(); // invalidated at each iteration
1464  symbol.erase(b + pos, b + symbol.find('\n', pos + 1) + 1);
1465  }
1466 
1467  std::map<std::string, std::string>::const_iterator arg;
1468  preproc_map::const_iterator macro;
1469 
1470  // If this is a known pre-processing symbol, then we insert it,
1471  // otherwise we assume it's a file name to load.
1472  if(symbol == current_file_str && strings_.size() - token.stack_pos == 1) {
1473  pop_token();
1475  } else if(symbol == current_dir_str && strings_.size() - token.stack_pos == 1) {
1476  pop_token();
1478  } else if(local_defines_ && (arg = local_defines_->find(symbol)) != local_defines_->end()) {
1479  if(strings_.size() - token.stack_pos != 1) {
1480  std::ostringstream error;
1481  error << "Macro argument '" << symbol << "' does not expect any arguments";
1482  parent_.error(error.str(), linenum_);
1483  }
1484 
1485  std::ostringstream v;
1486  v << arg->second << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1487  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1488 
1489  pop_token();
1490  put(v.str());
1491  } else if(parent_.depth() < 100 && (macro = parent_.defines_->find(symbol)) != parent_.defines_->end()) {
1492  const preproc_define& val = macro->second;
1493  std::size_t nb_arg = strings_.size() - token.stack_pos - 1;
1494  std::size_t optional_arg_num = 0;
1495 
1496  std::unique_ptr<std::map<std::string, std::string>> defines{new std::map<std::string, std::string>};
1497  const std::string& dir = filesystem::directory_name(val.location.substr(0, val.location.find(' ')));
1498 
1499  if(val.is_deprecated()) {
1501  }
1502 
1503  for(std::size_t i = 0; i < nb_arg; ++i) {
1504  if(i < val.arguments.size()) {
1505  // Normal mandatory arguments
1506 
1507  (*defines)[val.arguments[i]] = strings_[token.stack_pos + i + 1];
1508  } else {
1509  // These should be optional argument overrides
1510 
1511  std::string str = strings_[token.stack_pos + i + 1];
1512  std::size_t equals_pos = str.find_first_of("=");
1513 
1514  if(equals_pos != std::string::npos) {
1515  std::size_t argname_pos = str.substr(0, equals_pos).find_last_of(" \n") + 1;
1516 
1517  std::string argname = str.substr(argname_pos, equals_pos - argname_pos);
1518 
1519  if(val.optional_arguments.find(argname) != val.optional_arguments.end()) {
1520  (*defines)[argname] = str.substr(equals_pos + 1);
1521 
1522  optional_arg_num++;
1523 
1524  DBG_PREPROC << "Found override for " << argname << " in call to macro " << symbol
1525  << "\n";
1526  } else {
1527  std::ostringstream warning;
1528  warning << "Unrecognized optional argument passed to macro '" << symbol << "': '"
1529  << argname << "'";
1530  parent_.warning(warning.str(), linenum_);
1531 
1532  optional_arg_num++; // To prevent the argument number check from blowing up
1533  }
1534  }
1535  }
1536  }
1537 
1538  // If the macro definition has any optional arguments, insert their defaults
1539  if(val.optional_arguments.size() > 0) {
1540  for(const auto& argument : val.optional_arguments) {
1541  if(defines->find(argument.first) == defines->end()) {
1542  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1543 
1545  std::istream in(buf.get());
1546 
1547  filesystem::scoped_istream buffer{new std::istringstream(argument.second)};
1548 
1549  std::unique_ptr<std::map<std::string, std::string>> temp_defines{new std::map<std::string, std::string>};
1550  temp_defines->insert(defines->begin(), defines->end());
1551 
1553  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(temp_defines), false);
1554 
1555  std::ostringstream res;
1556  res << in.rdbuf();
1557 
1558  DBG_PREPROC << "Setting default for optional argument " << argument.first << " in macro "
1559  << symbol << "\n";
1560 
1561  (*defines)[argument.first] = res.str();
1562  }
1563  }
1564  }
1565 
1566  if(nb_arg - optional_arg_num != val.arguments.size()) {
1567  const std::vector<std::string>& locations = utils::quoted_split(val.location, ' ');
1568  std::ostringstream error;
1569  error << "Preprocessor symbol '" << symbol << "' defined at " << get_filename(locations[0]) << ":"
1570  << val.linenum << " expects " << val.arguments.size() << " arguments, but has "
1571  << nb_arg - optional_arg_num << " arguments";
1572  parent_.error(error.str(), linenum_);
1573  }
1574 
1575  filesystem::scoped_istream buffer{new std::istringstream(val.value)};
1576 
1577  pop_token();
1578 
1579  if(!slowpath_) {
1580  DBG_PREPROC << "substituting macro " << symbol << '\n';
1581 
1583  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1584  } else {
1585  DBG_PREPROC << "substituting (slow) macro " << symbol << '\n';
1586 
1587  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1588 
1589  // Make the nested preprocessor_data responsible for
1590  // restoring our current textdomain if needed.
1592 
1593  std::ostringstream res;
1594  {
1595  std::istream in(buf.get());
1597  std::move(buffer), val.location, "", val.linenum, dir, val.textdomain, std::move(defines), true);
1598 
1599  res << in.rdbuf();
1600  }
1601 
1602  put(res.str());
1603  }
1604  } else if(parent_.depth() < 40) {
1605  LOG_PREPROC << "Macro definition not found for " << symbol << " , attempting to open as file.\n";
1606  pop_token();
1607 
1608  std::string nfname = filesystem::get_wml_location(symbol, directory_);
1609  if(!nfname.empty()) {
1610  if(!slowpath_)
1611  // nfname.size() - symbol.size() gives you an index into nfname
1612  // This does not necessarily match the symbol though, as it can start with ~ or ./
1613  parent_.add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1614  else {
1615  std::unique_ptr<preprocessor_streambuf> buf(new preprocessor_streambuf(parent_));
1616 
1617  std::ostringstream res;
1618  {
1619  std::istream in(buf.get());
1620  buf->add_preprocessor<preprocessor_file>(nfname, nfname.size() - symbol.size());
1621 
1622  res << in.rdbuf();
1623  }
1624 
1625  put(res.str());
1626  }
1627  } else {
1628  std::ostringstream error;
1629  error << "Macro/file '" << symbol << "' is missing";
1630  parent_.error(error.str(), linenum_);
1631  }
1632  } else {
1633  parent_.error("Too many nested preprocessing inclusions", linenum_);
1634  }
1635  } else if(!skipping_) {
1636  if(token.type == token_desc::MACRO_SPACE) {
1637  std::ostringstream s;
1638  s << OUTPUT_SEPARATOR << "line " << linenum_ << ' ' << parent_.location_ << "\n"
1639  << OUTPUT_SEPARATOR << "textdomain " << parent_.textdomain_ << '\n';
1640 
1641  strings_.push_back(s.str());
1642  token.type = token_desc::MACRO_CHUNK;
1643  }
1644  put(c);
1645  }
1646  } else {
1647  put(c);
1648  }
1649 
1650  return true;
1651 }
1652 
1653 
1654 // ==================================================================================
1655 // PREPROCESSOR SCOPE HELPER
1656 // ==================================================================================
1657 
1658 struct preprocessor_scope_helper : std::basic_istream<char>
1659 {
1660  preprocessor_scope_helper(const std::string& fname, preproc_map* defines)
1661  : std::basic_istream<char>(nullptr)
1662  , buf_(nullptr)
1663  , local_defines_(nullptr)
1664  {
1665  //
1666  // If no defines were provided, we create a new local preproc_map and assign
1667  // it to defines temporarily. In this case, the map will be deleted once this
1668  // object is destroyed and defines will still be subsequently null.
1669  //
1670  if(!defines) {
1671  local_defines_.reset(new preproc_map);
1672  defines = local_defines_.get();
1673  }
1674 
1675  buf_.reset(new preprocessor_streambuf(defines));
1676 
1677  // Begin processing.
1678  buf_->add_preprocessor<preprocessor_file>(fname);
1679 
1680  //
1681  // TODO: not sure if this call is needed. Previously, this call was passed a
1682  // preprocessor_streambuf pointer and the std::basic_istream constructor was
1683  // called with its contents. However, at that point the preprocessing should
1684  // already have completed, meaning this call might be redundant. Not sure.
1685  //
1686  // - vultraz, 2017-08-31
1687  //
1688  init(buf_.get());
1689  }
1690 
1692  {
1693  clear(std::ios_base::goodbit);
1694  exceptions(std::ios_base::goodbit);
1695  rdbuf(nullptr);
1696  }
1697 
1698  std::unique_ptr<preprocessor_streambuf> buf_;
1699  std::unique_ptr<preproc_map> local_defines_;
1700 };
1701 
1702 
1703 // ==================================================================================
1704 // FREE-STANDING FUNCTIONS
1705 // ==================================================================================
1706 
1707 filesystem::scoped_istream preprocess_file(const std::string& fname, preproc_map* defines)
1708 {
1709  log_scope("preprocessing file " + fname + " ...");
1710 
1711  // NOTE: the preprocessor_scope_helper does *not* take ownership of defines.
1712  return filesystem::scoped_istream(new preprocessor_scope_helper(fname, defines));
1713 }
1714 
1715 void preprocess_resource(const std::string& res_name,
1716  preproc_map* defines_map,
1717  bool write_cfg,
1718  bool write_plain_cfg,
1719  const std::string& parent_directory)
1720 {
1721  if(filesystem::is_directory(res_name)) {
1722  std::vector<std::string> dirs, files;
1723 
1726 
1727  // Subdirectories
1728  for(const std::string& dir : dirs) {
1729  LOG_PREPROC << "processing sub-dir: " << dir << '\n';
1730  preprocess_resource(dir, defines_map, write_cfg, write_plain_cfg, parent_directory);
1731  }
1732 
1733  // Files in current directory
1734  for(const std::string& file : files) {
1735  preprocess_resource(file, defines_map, write_cfg, write_plain_cfg, parent_directory);
1736  }
1737 
1738  return;
1739  }
1740 
1741  // process only config files.
1742  if(!filesystem::ends_with(res_name, ".cfg")) {
1743  return;
1744  }
1745 
1746  LOG_PREPROC << "processing resource: " << res_name << '\n';
1747 
1748  // disable filename encoding to get clear #line in cfg.plain
1749  encode_filename = false;
1750 
1751  filesystem::scoped_istream stream = preprocess_file(res_name, defines_map);
1752 
1753  std::stringstream ss;
1754 
1755  // Set the failbit so if we get any preprocessor exceptions (e.g.:preproc_config::error)
1756  // they will be propagated in the main program, instead of just setting the
1757  // failbit on the stream. This was necessary in order for the MSVC and GCC
1758  // binaries to behave the same way.
1759  ss.exceptions(std::ios_base::failbit);
1760 
1761  ss << (*stream).rdbuf();
1762 
1763  LOG_PREPROC << "processing finished\n";
1764 
1765  if(write_cfg || write_plain_cfg) {
1766  config cfg;
1767  std::string streamContent = ss.str();
1768 
1769  read(cfg, streamContent);
1770 
1771  const std::string preproc_res_name = parent_directory + "/" + filesystem::base_name(res_name);
1772 
1773  // Write the processed cfg file
1774  if(write_cfg) {
1775  LOG_PREPROC << "writing cfg file: " << preproc_res_name << '\n';
1776 
1778  filesystem::scoped_ostream outStream(filesystem::ostream_file(preproc_res_name));
1779 
1780  write(*outStream, cfg);
1781  }
1782 
1783  // Write the plain cfg file
1784  if(write_plain_cfg) {
1785  LOG_PREPROC << "writing plain cfg file: " << (preproc_res_name + ".plain") << '\n';
1786 
1788  filesystem::write_file(preproc_res_name + ".plain", streamContent);
1789  }
1790  }
1791 }
static std::string current_file_str
std::string lineno_string(const std::string &lineno)
static const char OUTPUT_SEPARATOR
std::vector< std::string > arguments
preprocessor * current() const
preprocessor_file(preprocessor_streambuf &t, const std::string &name, std::size_t symbol_index=-1)
Constructor.
void write(config_writer &, const std::string &) const
std::vector< std::string > files_
virtual bool get_chunk() override
Inserts and processes the next file in the list of included files.
VERSION_COMP_OP parse_version_op(const std::string &op_str)
Definition: version.cpp:222
preprocessor_scope_helper(const std::string &fname, preproc_map *defines)
Specialized preprocessor for handling a file or a set of files.
void read(const config &)
static int read_line(lua_State *L, FILE *f, int chop)
Definition: liolib.cpp:477
std::vector< std::string >::const_iterator end_
std::string read_word()
preproc_map default_defines_
static l_noret error(LoadState *S, const char *why)
Definition: lundump.cpp:39
buffered_istream in_
Input stream.
static std::map< std::string, int > file_number_map
bool ends_with(const std::string &str, const std::string &suffix)
#define ERR_PREPROC
child_itors child_range(config_key_type key)
Definition: config.cpp:366
virtual MODE parse_mode()
Returns the appropriate parsing mode for this preprocessor.
int peek()
Gets a character from the buffer.
Specialized preprocessor for handling any kind of input stream.
std::string out_buffer_
Buffer read by the STL stream.
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
Definition: filesystem.cpp:894
void clear(const std::string &key)
Definition: general.cpp:207
Helper class for buffering a std::istream.
virtual bool get_chunk() override
Preprocesses and sends some text to the parent_ buffer.
void preprocess_resource(const std::string &res_name, preproc_map *defines_map, bool write_cfg, bool write_plain_cfg, const std::string &parent_directory)
STL namespace.
void read_argument(const config &)
const std::vector< std::string > items
#define d
std::unique_ptr< preprocessor_streambuf > buf_
Definitions for the interface to Wesnoth Markup Language (WML).
std::vector< std::string > split(const std::string &val, const char c, const int flags)
Splits a (comma-)separated string into a vector of pieces.
std::vector< std::string >::const_iterator pos_
std::string location
Description of a preprocessing chunk.
#define WRN_PREPROC
#define b
void write_argument(config_writer &, const std::string &) const
unsigned in
If equal to search_counter, the node is off the list.
void write(std::ostream &out, const configr_of &cfg, unsigned int level)
Definition: parser.cpp:749
int slowpath_
Set to true whenever input tokens cannot be directly sent to the target buffer.
bool is_define_
True iff we are currently parsing a macros content, otherwise false.
void write_key_val(const std::string &key, const T &value)
This template function will work with any type that can be assigned to an attribute_value.
int get()
Gets and consumes a character from the buffer.
void conditional_skip(bool skip)
std::string deprecated_message(const std::string &elem_name, DEP_LEVEL level, const version_info &version, const std::string &detail)
Definition: deprecation.cpp:29
bool quoted_
Set to true if one preprocessor for this target started to read a string.
void close_child(const std::string &key)
void add_preprocessor(A &&... args)
static lg::log_domain log_preprocessor("preprocessor")
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:612
Class for writing a config out to a file in pieces.
version_info deprecation_version
void restore_old_preprocessor()
Restores the old preprocessing context.
std::string read_line()
void write_file(const std::string &fname, const std::string &data)
Throws io_exception if an error occurs.
Definition: filesystem.cpp:955
int skipping_
Non-zero when the preprocessor has to skip some input text.
void push_token(token_desc::TOKEN_TYPE)
void open_child(const std::string &key)
std::deque< std::unique_ptr< preprocessor > > preprocessor_queue_
Input preprocessor queue.
virtual ~preprocessor()
bool is_deprecated() const
bool is_directory(const std::string &fname)
Returns true if the given file is a directory.
Definition: filesystem.cpp:984
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:39
virtual void init() override
Allows specifying any actions that need to be called after the constructor completes.
static bool create_directory_if_missing_recursive(const fs::path &dirpath)
Definition: filesystem.cpp:358
std::string get_short_wml_path(const std::string &filename)
Returns a short path to filename, skipping the (user) data directory.
std::string deprecation_message
#define DBG_PREPROC
std::string value
#define LOG_PREPROC
std::string escape(const std::string &str, const char *special_chars)
Prepends a configurable set of characters with a backslash.
#define PACKAGE
Definition: wesconfig.h:23
std::string directory_
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:40
virtual preprocessor::MODE parse_mode() override
Returns the appropriate parsing mode for this preprocessor.
static std::string get_file_code(const std::string &filename)
Some defines: VERSION, PACKAGE, MIN_SAVEGAME_VERSION.
virtual void init()
Allows specifying any actions that need to be called after the constructor completes.
bool operator==(const preproc_define &) const
token_desc(TOKEN_TYPE type, const int stack_pos, const int linenum)
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, file_name_option mode, file_filter_option filter, file_reorder_option reorder, file_tree_checksum *checksum)
Populates &#39;files&#39; with all the files and &#39;dirs&#39; with all the directories in dir.
Definition: filesystem.cpp:386
boost::optional< DEP_LEVEL > deprecation_level
std::size_t i
Definition: function.cpp:933
VERSION_COMP_OP
Definition: version.hpp:195
logger & err()
Definition: log.cpp:78
std::string get_wml_location(const std::string &filename, const std::string &current_dir)
Returns a complete path to the actual WML file or directory or an empty string if the file isn&#39;t pres...
std::string read_rest_of_line()
std::unique_ptr< preproc_map > local_defines_
bool operator!=(preprocessor_data::token_desc::TOKEN_TYPE rhs, char lhs)
mock_party p
static std::string get_location(const std::string &loc)
static map_location::DIRECTION s
filesystem::scoped_istream in_scope_
Manages the lifetime of the std::istream pointer we own.
std::vector< std::string > quoted_split(const std::string &val, char c, int flags, char quote)
This function is identical to split(), except it does not split when it otherwise would if the previo...
friend class preprocessor_streambuf
#define log_scope(description)
Definition: log.hpp:186
Helper class for buffering a std::istream.
std::unique_ptr< std::map< std::string, std::string > > local_defines_
Mapping of macro arguments to their content.
Declarations for File-IO.
static int writer(lua_State *L, const void *b, size_t size, void *B)
Definition: lstrlib.cpp:182
void warning(const std::string &, int)
const version_info wesnoth_version(VERSION)
Definition: version.hpp:210
Base class for preprocessing an input.
std::vector< std::string > strings_
Buffer for delayed input processing.
Represents version numbers.
Definition: version.hpp:43
std::ostream & operator<<(std::ostream &stream, const preproc_define &def)
static std::string current_dir_str
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
preprocessor_streambuf(const preprocessor_streambuf &t)
static std::string get_filename(const std::string &file_code)
The paths manager is responsible for recording the various paths that binary files may be located at...
Definition: fs_commit.cpp:37
double t
Definition: astarsearch.cpp:63
DEP_LEVEL
See https://wiki.wesnoth.org/CompatibilityStandards for more info.
Definition: deprecation.hpp:19
std::vector< token_desc > tokens_
Stack of nested preprocessing chunks.
preprocessor_streambuf & parent_
Standard logging facilities (interface).
std::string textdomain
std::string str() const
Serializes the version number into string form.
Definition: version.cpp:115
virtual int underflow() override
Inherited from basic_streambuf.
virtual bool get_chunk()=0
Preprocesses and sends some text to the parent_ buffer.
int stack_pos
Starting position in strings_ of the delayed text for this chunk.
preprocessor_data(preprocessor_streambuf &, filesystem::scoped_istream, const std::string &history, const std::string &name, int line, const std::string &dir, const std::string &domain, std::unique_ptr< std::map< std::string, std::string >> defines, bool is_define=false)
void error(const std::string &, int)
Target for sending preprocessed output.
std::map< std::string, struct preproc_define > preproc_map
#define e
bool do_version_check(const version_info &a, VERSION_COMP_OP op, const version_info &b)
Definition: version.cpp:241
bool portable_isspace(const char c)
preprocessor_streambuf(preproc_map *def)
bool operator==(preprocessor_data::token_desc::TOKEN_TYPE, char)
std::string get_current_file()
Decodes the filenames placed in a location.
std::string old_location_
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:68
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
static preproc_map::value_type read_pair(const config &)
mock_char c
const std::string & name_
std::string old_textdomain_
static map_location::DIRECTION n
bool operator<(const preproc_define &) const
static bool encode_filename
std::string::const_iterator iterator
Definition: tokenizer.hpp:24
std::stringstream buffer_
Buffer filled by the #current_ preprocessor.
preprocessor(preprocessor_streambuf &t)
Sets up a new preprocessor for stream buffer t.
std::string directory_name(const std::string &file)
Returns the directory name of a file, with filename stripped.
bool eof() const
Is the end of input reached?
static std::string preprocessor_error_detail_prefix
filesystem::scoped_ostream ostream_file(const std::string &fname, bool create_directory)
Definition: filesystem.cpp:932
std::map< std::string, std::string > optional_arguments