The Battle for Wesnoth  1.15.0-dev
config_cache.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2008 - 2018 by Pauli Nieminen <paniemin@cc.hut.fi>
3  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
4 
5  This program is free software; you can redistribute it and/or modify
6  it under the terms of the GNU General Public License as published by
7  the Free Software Foundation; either version 2 of the License, or
8  (at your option) any later version.
9  This program is distributed in the hope that it will be useful,
10  but WITHOUT ANY WARRANTY.
11 
12  See the COPYING file for more details.
13 */
14 
15 #define GETTEXT_DOMAIN "wesnoth-lib"
16 
17 #include "config_cache.hpp"
18 #include "filesystem.hpp"
19 #include "gettext.hpp"
20 #include "game_config.hpp"
21 #include "log.hpp"
22 #include "hash.hpp"
24 #include "serialization/parser.hpp"
26 #include "game_version.hpp"
27 
28 #include <boost/algorithm/string/replace.hpp>
29 #include <boost/iostreams/filter/gzip.hpp>
30 #include <SDL_platform.h>
31 
32 static lg::log_domain log_cache("cache");
33 #define ERR_CACHE LOG_STREAM(err, log_cache)
34 #define LOG_CACHE LOG_STREAM(info, log_cache)
35 #define DBG_CACHE LOG_STREAM(debug, log_cache)
36 
37 namespace game_config
38 {
39 
40 namespace
41 {
42 
43 void add_builtin_defines(preproc_map& target)
44 {
45 #ifdef __APPLE__
46  target["APPLE"] = preproc_define();
47 #endif
48 
49 #if defined(MOUSE_TOUCH_EMULATION) || defined(__IPHONEOS__)
50  target["IPHONEOS"] = preproc_define();
51 #endif
52 
53  target["WESNOTH_VERSION"] = preproc_define(game_config::wesnoth_version.str());
54 }
55 
56 }
57 
59 {
60  static config_cache cache;
61  return cache;
62 }
63 
65  : force_valid_cache_(false)
66  , use_cache_(true)
67  , fake_invalid_cache_(false)
68  , defines_map_()
69  , cache_file_prefix_("cache-v" + boost::algorithm::replace_all_copy(game_config::revision, ":", "_") + "-")
70 {
71  // To set-up initial defines map correctly
72  clear_defines();
73 }
74 
76 {
77  return defines_map_;
78 }
79 
81 {
82  LOG_CACHE << "Clearing defines map!" << std::endl;
83 
84  defines_map_.clear();
85 
86  //
87  // Set-up default defines map.
88  //
89 
90  add_builtin_defines(defines_map_);
91 }
92 
93 void config_cache::get_config(const std::string& file_path, config& cfg, abstract_validator* validator)
94 {
95  load_configs(file_path, cfg, validator);
96 }
97 
98 void config_cache::write_file(std::string file_path, const config& cfg)
99 {
102  writer.write(cfg);
103 }
104 
105 void config_cache::write_file(std::string file_path, const preproc_map& defines)
106 {
107  if(defines.empty()) {
108  if(filesystem::file_exists(file_path)) {
109  filesystem::delete_directory(file_path);
110  }
111  return;
112  }
113 
116 
117  // Write all defines to stream.
118  for(const preproc_map::value_type& define : defines) {
119  define.second.write(writer, define.first);
120  }
121 }
122 
123 void config_cache::read_file(const std::string& file_path, config& cfg)
124 {
126  read_gz(cfg, *stream);
127 }
128 
130 {
132  // HACK: copy_map doesn't have built-in defines in some cases (issue #1924)
133  // or they may be out of date, so brute-force them in.
134  add_builtin_defines(res);
135  return res;
136 }
137 
139 {
141 }
142 
143 void config_cache::read_configs(const std::string& file_path, config& cfg, preproc_map& defines_map, abstract_validator* validator)
144 {
145  //read the file and then write to the cache
146  filesystem::scoped_istream stream = preprocess_file(file_path, &defines_map);
147  read(cfg, *stream, validator);
148 }
149 
150 void config_cache::read_cache(const std::string& file_path, config& cfg, abstract_validator* validator)
151 {
152  static const std::string extension = ".gz";
153 
154  std::stringstream defines_string;
155  defines_string << file_path;
156 
157  bool is_valid = true;
158 
159  for(const preproc_map::value_type& d : defines_map_) {
160  //
161  // Only WESNOTH_VERSION is allowed to be non-empty.
162  //
163  if((!d.second.value.empty() || !d.second.arguments.empty()) &&
164  d.first != "WESNOTH_VERSION")
165  {
166  is_valid = false;
167  ERR_CACHE << "Invalid preprocessor define: " << d.first << '\n';
168  break;
169  }
170 
171  defines_string << " " << d.first;
172  }
173 
174  // Do cache check only if define map is valid and
175  // caching is allowed.
176  const std::string& cache_path = filesystem::get_cache_dir();
177 
178  if(is_valid && !cache_path.empty()) {
179  // Use a hash for a shorter display of the defines.
180  const std::string fname = cache_path + "/" +
182  utils::sha1(defines_string.str()).hex_digest();
183  const std::string fname_checksum = fname + ".checksum" + extension;
184 
185  filesystem::file_tree_checksum dir_checksum;
186 
188  try {
189  if(filesystem::file_exists(fname_checksum)) {
190  config checksum_cfg;
191 
192  DBG_CACHE << "Reading checksum: " << fname_checksum << "\n";
193  read_file(fname_checksum, checksum_cfg);
194 
195  dir_checksum = filesystem::file_tree_checksum(checksum_cfg);
196  }
197  } catch(const config::error&) {
198  ERR_CACHE << "cache checksum is corrupt" << std::endl;
199  } catch(const filesystem::io_exception&) {
200  ERR_CACHE << "error reading cache checksum" << std::endl;
201  }
202  }
203 
204  if(force_valid_cache_) {
205  LOG_CACHE << "skipping cache validation (forced)\n";
206  }
207 
208  if(filesystem::file_exists(fname + extension) && (force_valid_cache_ || (dir_checksum == filesystem::data_tree_checksum()))) {
209  LOG_CACHE << "found valid cache at '" << fname << extension << "' with defines_map " << defines_string.str() << "\n";
210  log_scope("read cache");
211 
212  try {
213  read_file(fname + extension,cfg);
214  const std::string define_file = fname + ".define" + extension;
215 
216  if(filesystem::file_exists(define_file)) {
218  }
219 
220  return;
221  } catch(const config::error& e) {
222  ERR_CACHE << "cache " << fname << extension << " is corrupt. Loading from files: "<< e.message << std::endl;
223  } catch(const filesystem::io_exception&) {
224  ERR_CACHE << "error reading cache " << fname << extension << ". Loading from files" << std::endl;
225  } catch (const boost::iostreams::gzip_error& e) {
226  //read_file -> ... -> read_gz can throw this exception.
227  ERR_CACHE << "cache " << fname << extension << " is corrupt. Error code: " << e.error() << std::endl;
228  }
229  }
230 
231  LOG_CACHE << "no valid cache found. Writing cache to '" << fname << extension << " with defines_map "<< defines_string.str() << "'\n";
232 
233  // Now we need queued defines so read them to memory
235 
236  preproc_map copy_map(make_copy_map());
237 
238  read_configs(file_path, cfg, copy_map, validator);
239  add_defines_map_diff(copy_map);
240 
241  try {
242  write_file(fname + extension, cfg);
243  write_file(fname + ".define" + extension, copy_map);
244 
245  config checksum_cfg;
246 
247  filesystem::data_tree_checksum().write(checksum_cfg);
248  write_file(fname_checksum, checksum_cfg);
249  } catch(const filesystem::io_exception&) {
250  ERR_CACHE << "could not write to cache '" << fname << "'" << std::endl;
251  }
252 
253  return;
254  }
255 
256  LOG_CACHE << "Loading plain config instead of cache\n";
257 
258  preproc_map copy_map(make_copy_map());
259  read_configs(file_path, cfg, copy_map, validator);
260  add_defines_map_diff(copy_map);
261 }
262 
263 void config_cache::read_defines_file(const std::string& file_path)
264 {
265  config cfg;
266  read_file(file_path, cfg);
267 
268  DBG_CACHE << "Reading cached defines from: " << file_path << "\n";
269 
270  // use static preproc_define::read_pair(config) to make a object
271  // and pass that object config_cache_transaction::insert_to_active method
272  for(const config::any_child &value : cfg.all_children_range()) {
274  preproc_define::read_pair(value.cfg));
275  }
276 }
277 
279 {
280  const std::vector<std::string>& files = config_cache_transaction::instance().get_define_files();
281 
282  for(const std::string &p : files) {
284  }
285 }
286 
287 void config_cache::load_configs(const std::string& config_path, config& cfg, abstract_validator* validator)
288 {
289  // Make sure that we have fake transaction if no real one is going on
290  fake_transaction fake;
291 
292  if (use_cache_) {
293  read_cache(config_path, cfg, validator);
294  } else {
295  preproc_map copy_map(make_copy_map());
296  read_configs(config_path, cfg, copy_map, validator);
297  add_defines_map_diff(copy_map);
298  }
299 }
300 
302 {
303  fake_invalid_cache_ = force;
304 }
305 
307 {
308  use_cache_ = use;
309 }
310 
312 {
313  force_valid_cache_ = force;
314 }
315 
317 {
319 }
320 
321 void config_cache::add_define(const std::string& define)
322 {
323  DBG_CACHE << "adding define: " << define << "\n";
324  defines_map_[define] = preproc_define();
325 
327  // we have to add this to active map too
329  }
330 
331 }
332 
333 void config_cache::remove_define(const std::string& define)
334 {
335  DBG_CACHE << "removing define: " << define << "\n";
336  defines_map_.erase(define);
337 
339  // we have to remove this from active map too
341  }
342 }
343 
345 {
346  std::vector<std::string> files, dirs;
348 
349  LOG_CACHE << "clean_cache(): " << files.size() << " files, "
350  << dirs.size() << " dirs to check\n";
351 
352  const std::string& exclude_current = cache_file_prefix_ + "*";
353 
354  bool status = true;
355 
356  status &= delete_cache_files(files, exclude_current);
357  status &= delete_cache_files(dirs, exclude_current);
358 
359  LOG_CACHE << "clean_cache(): done\n";
360 
361  return status;
362 }
363 
365 {
366  std::vector<std::string> files, dirs;
368 
369  LOG_CACHE << "purge_cache(): deleting " << files.size() << " files, "
370  << dirs.size() << " dirs\n";
371 
372  bool status = true;
373 
374  status &= delete_cache_files(files);
375  status &= delete_cache_files(dirs);
376 
377  LOG_CACHE << "purge_cache(): done\n";
378  return status;
379 }
380 
381 bool config_cache::delete_cache_files(const std::vector<std::string>& paths,
382  const std::string& exclude_pattern)
383 {
384  const bool delete_everything = exclude_pattern.empty();
385  bool status = true;
386 
387  for(const std::string& file_path : paths)
388  {
389  if(!delete_everything) {
390  const std::string& fn = filesystem::base_name(file_path);
391 
392  if(utils::wildcard_string_match(fn, exclude_pattern)) {
393  LOG_CACHE << "delete_cache_files(): skipping " << file_path
394  << " excluded by '" << exclude_pattern << "'\n";
395  continue;
396  }
397  }
398 
399  LOG_CACHE << "delete_cache_files(): deleting " << file_path << '\n';
400  if(!filesystem::delete_directory(file_path)) {
401  ERR_CACHE << "delete_cache_files(): could not delete "
402  << file_path << '\n';
403  status = false;
404  }
405  }
406 
407  return status;
408 }
409 
412 
414  : define_filenames_()
415  , active_map_()
416 {
417  assert(state_ == FREE);
418  state_ = NEW;
419  active_ = this;
420 }
421 
423 {
424  state_ = FREE;
425  active_ = 0;
426 }
427 
429 {
430  state_ = LOCKED;
431 }
432 
433 const std::vector<std::string>& config_cache_transaction::get_define_files() const
434 {
435  return define_filenames_;
436 }
437 
438 void config_cache_transaction::add_define_file(const std::string& file)
439 {
440  define_filenames_.push_back(file);
441 }
442 
444 {
445  if(active_map_.empty()) {
446  active_map_.insert(defines_map.begin(), defines_map.end());
447  if(get_state() == NEW) {
448  state_ = ACTIVE;
449  }
450  }
451 
452  return active_map_;
453 }
454 
455 namespace
456 {
457 
458 bool compare_define(const preproc_map::value_type& a, const preproc_map::value_type& b)
459 {
460  if(a.first < b.first) {
461  return true;
462  }
463 
464  if(b.first < a.first) {
465  return false;
466  }
467 
468  if(a.second < b.second) {
469  return true;
470  }
471 
472  return false;
473 }
474 
475 } // end anonymous namespace
476 
478 {
479  if(get_state() == ACTIVE) {
480  preproc_map temp;
481  std::set_difference(new_map.begin(),
482  new_map.end(),
483  active_map_.begin(),
484  active_map_.end(),
485  std::insert_iterator<preproc_map>(temp,temp.begin()),
486  &compare_define);
487 
488  for(const preproc_map::value_type &def : temp) {
489  insert_to_active(def);
490  }
491 
492  temp.swap(new_map);
493  } else if (get_state() == LOCKED) {
494  new_map.clear();
495  }
496 }
497 
498 void config_cache_transaction::insert_to_active(const preproc_map::value_type& def)
499 {
500  active_map_[def.first] = def.second;
501 }
502 
503 }
bool delete_directory(const std::string &dirname, const bool keep_pbl)
Definition: filesystem.cpp:860
static config_cache & instance()
Get reference to the singleton object.
const_all_children_itors all_children_range() const
In-order iteration over all children.
Definition: config.cpp:921
void write(const config &cfg)
void write_file(std::string file, const config &cfg)
Interfaces for manipulating version numbers of engine, add-ons, etc.
void read_file(const std::string &file, config &cfg)
#define a
void lock()
Lock the transaction so no more macros are added.
static bool file_exists(const bfs::path &fpath)
Definition: filesystem.cpp:280
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
Definition: filesystem.cpp:918
void read_cache(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
bool wildcard_string_match(const std::string &str, const std::string &match)
Match using &#39;*&#39; as any number of characters (including none), &#39;+&#39; as one or more characters, and &#39;?&#39; as any one character.
static config_cache_transaction * active_
void remove_define(const std::string &define)
Remove a entry to preproc defines map.
Holds a fake cache transaction if no real one is used.
#define d
std::vector< std::string > define_filenames_
void read_gz(config &cfg, std::istream &file, abstract_validator *validator)
Might throw a std::ios_base::failure especially a gzip_error.
Definition: parser.cpp:681
static config_cache_transaction & instance()
const std::vector< std::string > & get_define_files() const
#define b
Used in parsing config file.
Definition: validator.hpp:35
bool delete_cache_files(const std::vector< std::string > &paths, const std::string &exclude_pattern="")
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:625
Class for writing a config out to a file in pieces.
bool clean_cache()
Deletes stale cache files not in use by the game.
void read_defines_file(const std::string &path)
void add_define_file(const std::string &file)
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:39
void set_force_valid_cache(bool force)
Enable/disable cache validation.
#define LOG_CACHE
void read_configs(const std::string &path, config &cfg, preproc_map &defines, abstract_validator *validator=nullptr)
static lg::log_domain log_cache("cache")
void get_config(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
Gets a config object from given path.
void set_use_cache(bool use)
Enable/disable caching.
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:40
std::string get_cache_dir()
Definition: filesystem.cpp:778
void clear_defines()
Clear stored defines map to default values.
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, file_name_option mode, file_filter_option filter, file_reorder_option reorder, file_tree_checksum *checksum)
Populates &#39;files&#39; with all the files and &#39;dirs&#39; with all the directories in dir.
Definition: filesystem.cpp:366
const std::string revision
void add_defines_map_diff(preproc_map &)
mock_party p
Game configuration data as global variables.
Definition: build_info.cpp:49
#define DBG_CACHE
const preproc_map & get_preproc_map() const
An exception object used when an IO error occurs.
Definition: filesystem.hpp:48
static tcache cache
Definition: minimap.cpp:134
preproc_map & get_active_map(const preproc_map &defines_map)
#define log_scope(description)
Definition: log.hpp:186
Declarations for File-IO.
static int writer(lua_State *L, const void *b, size_t size, void *B)
Definition: lstrlib.cpp:182
const version_info wesnoth_version(VERSION)
const file_tree_checksum & data_tree_checksum(bool reset=false)
Get the time at which the data/ tree was last modified at.
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
Used to share macros between cache objects You have to create transaction object to load all macros t...
#define ERR_CACHE
void load_configs(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
void add_defines_map_diff(preproc_map &defines_map)
Standard logging facilities (interface).
std::string message
Definition: exceptions.hpp:31
std::map< std::string, struct preproc_define > preproc_map
#define e
void insert_to_active(const preproc_map::value_type &def)
Used to let std::for_each insert new defines to active_map map to active.
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:92
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
static preproc_map::value_type read_pair(const config &)
preproc_map & make_copy_map()
void add_define(const std::string &define)
Add a entry to preproc defines map.
void recheck_filetree_checksum()
Force cache checksum validation.
bool purge_cache()
Deletes all cache files.
Singleton class to manage game config file caching.
filesystem::scoped_ostream ostream_file(const std::string &fname, bool create_directory)
Definition: filesystem.cpp:956
int cache_compression_level
Definition: game_config.cpp:85