The Battle for Wesnoth  1.15.12+dev
config_cache.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2008 - 2018 by Pauli Nieminen <paniemin@cc.hut.fi>
3  Part of the Battle for Wesnoth Project https://www.wesnoth.org/
4 
5  This program is free software; you can redistribute it and/or modify
6  it under the terms of the GNU General Public License as published by
7  the Free Software Foundation; either version 2 of the License, or
8  (at your option) any later version.
9  This program is distributed in the hope that it will be useful,
10  but WITHOUT ANY WARRANTY.
11 
12  See the COPYING file for more details.
13 */
14 
15 #define GETTEXT_DOMAIN "wesnoth-lib"
16 
17 #include "config_cache.hpp"
18 #include "filesystem.hpp"
19 #include "gettext.hpp"
20 #include "game_config.hpp"
21 #include "log.hpp"
22 #include "hash.hpp"
24 #include "serialization/parser.hpp"
26 #include "game_version.hpp"
27 
28 #include <boost/algorithm/string/replace.hpp>
29 #include <boost/iostreams/filter/gzip.hpp>
30 
31 static lg::log_domain log_cache("cache");
32 #define ERR_CACHE LOG_STREAM(err, log_cache)
33 #define LOG_CACHE LOG_STREAM(info, log_cache)
34 #define DBG_CACHE LOG_STREAM(debug, log_cache)
35 
36 namespace game_config
37 {
38 
39 namespace
40 {
41 
42 void add_builtin_defines(preproc_map& target)
43 {
44 #ifdef __APPLE__
45  target["APPLE"] = preproc_define();
46 #endif
47 
48 #if defined(MOUSE_TOUCH_EMULATION) || defined(TARGET_OS_IPHONE)
49  target["IPHONEOS"] = preproc_define();
50 #endif
51 
52  target["WESNOTH_VERSION"] = preproc_define(game_config::wesnoth_version.str());
53 }
54 
55 }
56 
58 {
59  static config_cache cache;
60  return cache;
61 }
62 
64  : force_valid_cache_(false)
65  , use_cache_(true)
66  , fake_invalid_cache_(false)
67  , defines_map_()
68  , cache_file_prefix_("cache-v" + boost::algorithm::replace_all_copy(game_config::revision, ":", "_") + "-")
69 {
70  // To set-up initial defines map correctly
71  clear_defines();
72 }
73 
75 {
76  return defines_map_;
77 }
78 
80 {
81  LOG_CACHE << "Clearing defines map!" << std::endl;
82 
83  defines_map_.clear();
84 
85  //
86  // Set-up default defines map.
87  //
88 
89  add_builtin_defines(defines_map_);
90 }
91 
92 void config_cache::get_config(const std::string& file_path, config& cfg, abstract_validator* validator)
93 {
94  load_configs(file_path, cfg, validator);
95 }
96 
97 void config_cache::write_file(std::string file_path, const config& cfg)
98 {
101  writer.write(cfg);
102 }
103 
104 void config_cache::write_file(std::string file_path, const preproc_map& defines)
105 {
106  if(defines.empty()) {
107  if(filesystem::file_exists(file_path)) {
108  filesystem::delete_directory(file_path);
109  }
110  return;
111  }
112 
115 
116  // Write all defines to stream.
117  for(const preproc_map::value_type& define : defines) {
118  define.second.write(writer, define.first);
119  }
120 }
121 
122 void config_cache::read_file(const std::string& file_path, config& cfg)
123 {
125  read_gz(cfg, *stream);
126 }
127 
129 {
131  // HACK: copy_map doesn't have built-in defines in some cases (issue #1924)
132  // or they may be out of date, so brute-force them in.
133  add_builtin_defines(res);
134  return res;
135 }
136 
138 {
140 }
141 
142 void config_cache::read_configs(const std::string& file_path, config& cfg, preproc_map& defines_map, abstract_validator* validator)
143 {
144  //read the file and then write to the cache
145  filesystem::scoped_istream stream = preprocess_file(file_path, &defines_map);
146  read(cfg, *stream, validator);
147 }
148 
149 void config_cache::read_cache(const std::string& file_path, config& cfg, abstract_validator* validator)
150 {
151  static const std::string extension = ".gz";
152 
153  std::stringstream defines_string;
154  defines_string << file_path;
155 
156  bool is_valid = true;
157 
158  for(const preproc_map::value_type& d : defines_map_) {
159  //
160  // Only WESNOTH_VERSION is allowed to be non-empty.
161  //
162  if((!d.second.value.empty() || !d.second.arguments.empty()) &&
163  d.first != "WESNOTH_VERSION")
164  {
165  is_valid = false;
166  ERR_CACHE << "Invalid preprocessor define: " << d.first << '\n';
167  break;
168  }
169 
170  defines_string << " " << d.first;
171  }
172 
173  // Do cache check only if define map is valid and
174  // caching is allowed.
175  const std::string& cache_path = filesystem::get_cache_dir();
176 
177  if(is_valid && !cache_path.empty()) {
178  // Use a hash for a shorter display of the defines.
179  const std::string fname = cache_path + "/" +
181  utils::sha1(defines_string.str()).hex_digest();
182  const std::string fname_checksum = fname + ".checksum" + extension;
183 
184  filesystem::file_tree_checksum dir_checksum;
185 
187  try {
188  if(filesystem::file_exists(fname_checksum)) {
189  config checksum_cfg;
190 
191  DBG_CACHE << "Reading checksum: " << fname_checksum << "\n";
192  read_file(fname_checksum, checksum_cfg);
193 
194  dir_checksum = filesystem::file_tree_checksum(checksum_cfg);
195  }
196  } catch(const config::error&) {
197  ERR_CACHE << "cache checksum is corrupt" << std::endl;
198  } catch(const filesystem::io_exception&) {
199  ERR_CACHE << "error reading cache checksum" << std::endl;
200  } catch(const std::ios_base::failure&) {
201  ERR_CACHE << "error reading cache checksum" << std::endl;
202  }
203  }
204 
205  if(force_valid_cache_) {
206  LOG_CACHE << "skipping cache validation (forced)\n";
207  }
208 
209  if(filesystem::file_exists(fname + extension) && (force_valid_cache_ || (dir_checksum == filesystem::data_tree_checksum()))) {
210  LOG_CACHE << "found valid cache at '" << fname << extension << "' with defines_map " << defines_string.str() << "\n";
211  log_scope("read cache");
212 
213  try {
214  read_file(fname + extension,cfg);
215  const std::string define_file = fname + ".define" + extension;
216 
217  if(filesystem::file_exists(define_file)) {
219  }
220 
221  return;
222  } catch(const config::error& e) {
223  ERR_CACHE << "cache " << fname << extension << " is corrupt. Loading from files: "<< e.message << std::endl;
224  } catch(const filesystem::io_exception&) {
225  ERR_CACHE << "error reading cache " << fname << extension << ". Loading from files" << std::endl;
226  } catch (const boost::iostreams::gzip_error& e) {
227  //read_file -> ... -> read_gz can throw this exception.
228  ERR_CACHE << "cache " << fname << extension << " is corrupt. Error code: " << e.error() << std::endl;
229  }
230  }
231 
232  LOG_CACHE << "no valid cache found. Writing cache to '" << fname << extension << " with defines_map "<< defines_string.str() << "'\n";
233 
234  // Now we need queued defines so read them to memory
236 
237  preproc_map copy_map(make_copy_map());
238 
239  read_configs(file_path, cfg, copy_map, validator);
240  add_defines_map_diff(copy_map);
241 
242  try {
243  write_file(fname + extension, cfg);
244  write_file(fname + ".define" + extension, copy_map);
245 
246  config checksum_cfg;
247 
248  filesystem::data_tree_checksum().write(checksum_cfg);
249  write_file(fname_checksum, checksum_cfg);
250  } catch(const filesystem::io_exception&) {
251  ERR_CACHE << "could not write to cache '" << fname << "'" << std::endl;
252  }
253 
254  return;
255  }
256 
257  LOG_CACHE << "Loading plain config instead of cache\n";
258 
259  preproc_map copy_map(make_copy_map());
260  read_configs(file_path, cfg, copy_map, validator);
261  add_defines_map_diff(copy_map);
262 }
263 
264 void config_cache::read_defines_file(const std::string& file_path)
265 {
266  config cfg;
267  read_file(file_path, cfg);
268 
269  DBG_CACHE << "Reading cached defines from: " << file_path << "\n";
270 
271  // use static preproc_define::read_pair(config) to make a object
272  // and pass that object config_cache_transaction::insert_to_active method
273  for(const config::any_child &value : cfg.all_children_range()) {
275  preproc_define::read_pair(value.cfg));
276  }
277 }
278 
280 {
281  const std::vector<std::string>& files = config_cache_transaction::instance().get_define_files();
282 
283  for(const std::string &p : files) {
285  }
286 }
287 
288 void config_cache::load_configs(const std::string& config_path, config& cfg, abstract_validator* validator)
289 {
290  // Make sure that we have fake transaction if no real one is going on
291  fake_transaction fake;
292 
293  if (use_cache_) {
294  read_cache(config_path, cfg, validator);
295  } else {
296  preproc_map copy_map(make_copy_map());
297  read_configs(config_path, cfg, copy_map, validator);
298  add_defines_map_diff(copy_map);
299  }
300 }
301 
303 {
304  fake_invalid_cache_ = force;
305 }
306 
308 {
309  use_cache_ = use;
310 }
311 
313 {
314  force_valid_cache_ = force;
315 }
316 
318 {
320 }
321 
322 void config_cache::add_define(const std::string& define)
323 {
324  DBG_CACHE << "adding define: " << define << "\n";
325  defines_map_[define] = preproc_define();
326 
328  // we have to add this to active map too
330  }
331 
332 }
333 
334 void config_cache::remove_define(const std::string& define)
335 {
336  DBG_CACHE << "removing define: " << define << "\n";
337  defines_map_.erase(define);
338 
340  // we have to remove this from active map too
342  }
343 }
344 
346 {
347  std::vector<std::string> files, dirs;
349 
350  LOG_CACHE << "clean_cache(): " << files.size() << " files, "
351  << dirs.size() << " dirs to check\n";
352 
353  const std::string& exclude_current = cache_file_prefix_ + "*";
354 
355  bool status = true;
356 
357  status &= delete_cache_files(files, exclude_current);
358  status &= delete_cache_files(dirs, exclude_current);
359 
360  LOG_CACHE << "clean_cache(): done\n";
361 
362  return status;
363 }
364 
366 {
367  std::vector<std::string> files, dirs;
369 
370  LOG_CACHE << "purge_cache(): deleting " << files.size() << " files, "
371  << dirs.size() << " dirs\n";
372 
373  bool status = true;
374 
375  status &= delete_cache_files(files);
376  status &= delete_cache_files(dirs);
377 
378  LOG_CACHE << "purge_cache(): done\n";
379  return status;
380 }
381 
382 bool config_cache::delete_cache_files(const std::vector<std::string>& paths,
383  const std::string& exclude_pattern)
384 {
385  const bool delete_everything = exclude_pattern.empty();
386  bool status = true;
387 
388  for(const std::string& file_path : paths)
389  {
390  if(!delete_everything) {
391  const std::string& fn = filesystem::base_name(file_path);
392 
393  if(utils::wildcard_string_match(fn, exclude_pattern)) {
394  LOG_CACHE << "delete_cache_files(): skipping " << file_path
395  << " excluded by '" << exclude_pattern << "'\n";
396  continue;
397  }
398  }
399 
400  LOG_CACHE << "delete_cache_files(): deleting " << file_path << '\n';
401  if(!filesystem::delete_directory(file_path)) {
402  ERR_CACHE << "delete_cache_files(): could not delete "
403  << file_path << '\n';
404  status = false;
405  }
406  }
407 
408  return status;
409 }
410 
413 
415  : define_filenames_()
416  , active_map_()
417 {
418  assert(state_ == FREE);
419  state_ = NEW;
420  active_ = this;
421 }
422 
424 {
425  state_ = FREE;
426  active_ = 0;
427 }
428 
430 {
431  state_ = LOCKED;
432 }
433 
434 const std::vector<std::string>& config_cache_transaction::get_define_files() const
435 {
436  return define_filenames_;
437 }
438 
439 void config_cache_transaction::add_define_file(const std::string& file)
440 {
441  define_filenames_.push_back(file);
442 }
443 
445 {
446  if(active_map_.empty()) {
447  active_map_.insert(defines_map.begin(), defines_map.end());
448  if(get_state() == NEW) {
449  state_ = ACTIVE;
450  }
451  }
452 
453  return active_map_;
454 }
455 
456 namespace
457 {
458 
459 bool compare_define(const preproc_map::value_type& a, const preproc_map::value_type& b)
460 {
461  if(a.first < b.first) {
462  return true;
463  }
464 
465  if(b.first < a.first) {
466  return false;
467  }
468 
469  if(a.second < b.second) {
470  return true;
471  }
472 
473  return false;
474 }
475 
476 } // end anonymous namespace
477 
479 {
480  if(get_state() == ACTIVE) {
481  preproc_map temp;
482  std::set_difference(new_map.begin(),
483  new_map.end(),
484  active_map_.begin(),
485  active_map_.end(),
486  std::insert_iterator<preproc_map>(temp,temp.begin()),
487  &compare_define);
488 
489  for(const preproc_map::value_type &def : temp) {
490  insert_to_active(def);
491  }
492 
493  temp.swap(new_map);
494  } else if (get_state() == LOCKED) {
495  new_map.clear();
496  }
497 }
498 
499 void config_cache_transaction::insert_to_active(const preproc_map::value_type& def)
500 {
501  active_map_[def.first] = def.second;
502 }
503 
504 }
bool delete_directory(const std::string &dirname, const bool keep_pbl)
Definition: filesystem.cpp:945
static config_cache & instance()
Get reference to the singleton object.
const_all_children_itors all_children_range() const
In-order iteration over all children.
Definition: config.cpp:953
void write(const config &cfg)
void write_file(std::string file, const config &cfg)
Interfaces for manipulating version numbers of engine, add-ons, etc.
void read_file(const std::string &file, config &cfg)
#define a
void lock()
Lock the transaction so no more macros are added.
static bool file_exists(const bfs::path &fpath)
Definition: filesystem.cpp:263
filesystem::scoped_istream istream_file(const std::string &fname, bool treat_failure_as_error)
void read_cache(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
bool wildcard_string_match(const std::string &str, const std::string &match)
Match using &#39;*&#39; as any number of characters (including none), &#39;+&#39; as one or more characters, and &#39;?&#39; as any one character.
static config_cache_transaction * active_
void remove_define(const std::string &define)
Remove a entry to preproc defines map.
Holds a fake cache transaction if no real one is used.
#define d
std::vector< std::string > define_filenames_
void read_gz(config &cfg, std::istream &file, abstract_validator *validator)
Might throw a std::ios_base::failure especially a gzip_error.
Definition: parser.cpp:682
static config_cache_transaction & instance()
filesystem::scoped_ostream ostream_file(const std::string &fname, std::ios_base::openmode mode, bool create_directory)
const std::vector< std::string > & get_define_files() const
#define b
Used in parsing config file.
Definition: validator.hpp:35
bool delete_cache_files(const std::vector< std::string > &paths, const std::string &exclude_pattern="")
void read(config &cfg, std::istream &in, abstract_validator *validator)
Definition: parser.cpp:626
Class for writing a config out to a file in pieces.
bool clean_cache()
Deletes stale cache files not in use by the game.
void read_defines_file(const std::string &path)
void get_files_in_dir(const std::string &dir, std::vector< std::string > *files, std::vector< std::string > *dirs, name_mode mode, filter_mode filter, reorder_mode reorder, file_tree_checksum *checksum)
Populates &#39;files&#39; with all the files and &#39;dirs&#39; with all the directories in dir.
Definition: filesystem.cpp:349
void add_define_file(const std::string &file)
std::unique_ptr< std::istream > scoped_istream
Definition: filesystem.hpp:37
void set_force_valid_cache(bool force)
Enable/disable cache validation.
#define LOG_CACHE
void read_configs(const std::string &path, config &cfg, preproc_map &defines, abstract_validator *validator=nullptr)
static lg::log_domain log_cache("cache")
static int writer(lua_State *L, const void *b, size_t size, void *ud)
Definition: lstrlib.cpp:221
void get_config(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
Gets a config object from given path.
void set_use_cache(bool use)
Enable/disable caching.
std::unique_ptr< std::ostream > scoped_ostream
Definition: filesystem.hpp:38
std::string get_cache_dir()
Definition: filesystem.cpp:794
void clear_defines()
Clear stored defines map to default values.
const std::string revision
void add_defines_map_diff(preproc_map &)
mock_party p
Game configuration data as global variables.
Definition: build_info.cpp:58
#define DBG_CACHE
const preproc_map & get_preproc_map() const
An exception object used when an IO error occurs.
Definition: filesystem.hpp:46
static tcache cache
Definition: minimap.cpp:123
preproc_map & get_active_map(const preproc_map &defines_map)
#define log_scope(description)
Definition: log.hpp:206
Declarations for File-IO.
const version_info wesnoth_version(VERSION)
const file_tree_checksum & data_tree_checksum(bool reset=false)
Get the time at which the data/ tree was last modified at.
std::string base_name(const std::string &file, const bool remove_extension)
Returns the base filename of a file, with directory name stripped.
Used to share macros between cache objects You have to create transaction object to load all macros t...
#define ERR_CACHE
void load_configs(const std::string &path, config &cfg, abstract_validator *validator=nullptr)
void add_defines_map_diff(preproc_map &defines_map)
Standard logging facilities (interface).
std::string message
Definition: exceptions.hpp:29
std::map< std::string, struct preproc_define > preproc_map
#define e
void insert_to_active(const preproc_map::value_type &def)
Used to let std::for_each insert new defines to active_map map to active.
A config object defines a single node in a WML file, with access to child nodes.
Definition: config.hpp:59
filesystem::scoped_istream preprocess_file(const std::string &fname, preproc_map *defines)
Function to use the WML preprocessor on a file.
static preproc_map::value_type read_pair(const config &)
preproc_map & make_copy_map()
void add_define(const std::string &define)
Add a entry to preproc defines map.
void recheck_filetree_checksum()
Force cache checksum validation.
bool purge_cache()
Deletes all cache files.
Singleton class to manage game config file caching.
int cache_compression_level
Definition: game_config.cpp:84